def construct_ellreg_v4_resnet(images, is_training):
batch_norm_params = {
'is_training': is_training,
'decay': 0.8,
'updates_collections': None,
'center': True,
'scale': True,
'trainable': True
}
input_imgs = tf.map_fn(lambda by_batch: concat_xygrad_2d(by_batch), images)
mynet, _ = resnet_v2.resnet_v2_200(input_imgs,
None,
is_training=is_training)
features = tf.reshape(mynet, [-1, 2048])
with slim.arg_scope([slim.fully_connected],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(
0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0005),
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
mynet = slim.fully_connected(features,
6,
activation_fn=None,
normalizer_fn=None,
normalizer_params=None,
scope='outlayer')
return mynet, features
def __restore_model(self):
with tf.device(self._Graph__device):
with self.__sess.as_default():
with self._Graph__graph.as_default():
self.images = tf.placeholder(
shape=[None, config.IMAGE_H, config.IMAGE_W, 1],
dtype=tf.float32)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
self.logits, _ = resnet_v2.resnet_v2_200(
self.images, config.NUM_CLASSES)
self.net = tf.squeeze(self.logits)
self.probs = tf.nn.softmax(self.net)
self.top_5_values, self.top_5_idx = tf.nn.top_k(
self.probs, 5)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
config.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore(
)
self.__saver_cls = tf.train.Saver(variables_to_restore)
print("All variables are loaded!")
# print("Meta-Graph Restored.")
ckpt = tf.train.get_checkpoint_state(
self._Graph__model_ckpt)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
self.__saver_cls.restore(self.__sess,
ckpt.model_checkpoint_path)
gbl_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
print("Model loaded successfully")
self._Graph__graph = tf.get_default_graph()
def model_fn(model_name, batch_size):
if model_name == "vgg19":
from tensorflow.contrib.slim.nets import vgg
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = vgg.vgg_19(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet200":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_200(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet101":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_101(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet152":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_152(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "nasnet_cifar":
from tensorflow.contrib.slim.nets import nasnet
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = nasnet.build_nasnet_cifar(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "mobile_net":
from tensorflow.contrib.slim.nets import mobilenet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = mobilenet_v2.mobilenet(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "inceptionv3":
from tensorflow.contrib.slim.nets import inception_v3
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = inception_v3.inception_v3(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "transformer":
import modeltransformer.transformer as transf
from modeltransformer.data import DatasetManager
dm = DatasetManager("wmt14")
dm.maybe_download_data_files()
dm.load_vocab()
transformer = transf.Transformer(
num_heads=8,
d_model=512,
d_ff=2048,
model_name=model_name,
tf_sess_config=dict(allow_soft_placement=True))
train_params = dict(
learning_rate=1e-4,
batch_size=batch_size,
seq_len=10,
max_steps=300000,
)
transformer.build_model("wmt14", dm.source_id2word, dm.target_id2word,
0, **train_params)
loss = transformer._loss
elif model_name == "bert":
from bert.runsquad import new_model_fn_builder
import modeling
bert_config = modeling.BertConfig.from_json_file(
"bert/bert_large/bert_config.json")
model = new_model_fn_builder(bert_config)
features = {}
features["input_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["input_mask"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["segment_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["start_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
features["end_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
loss = model(features)
elif model_name == "small":
slim = tf.contrib.slim
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
v = tf.get_variable(name="large_variable",
shape=(3000, 224, 224, 3),
trainable=True)
x = tf.slice(v, [0, 0, 0, 0], tf.shape(x), name="large_slice")
net = slim.max_pool2d(x, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.flatten(net)
net = slim.fully_connected(net,
1024,
activation_fn=tf.nn.sigmoid,
trainable=False)
net = slim.fully_connected(net,
1000,
activation_fn=None,
trainable=False)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=net)
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9,
beta2=0.98,
epsilon=1e-9).minimize(
tf.reduce_sum(loss))
return optimizer
def model_fn(model_name, batch_size):
if model_name=='vgg19':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = vgg.vgg_19(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet200':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_200(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet101':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_101(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet152':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_152(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='nasnet_cifar':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = nasnet.build_nasnet_cifar(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='mobile_net':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = mobilenet_v2.mobilenet(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='inceptionv3':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = inception.inception_v3(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='transformer':
dm = DatasetManager('wmt14')
dm.maybe_download_data_files()
dm.load_vocab()
transformer = transf.Transformer(
num_heads=8,
d_model=512,
d_ff=2048,
model_name=model_name,
tf_sess_config=dict(allow_soft_placement=True)
)
train_params = dict(
learning_rate=1e-4,
batch_size=batch_size,
seq_len=10,
max_steps=300000,
)
transformer.build_model('wmt14', dm.source_id2word, dm.target_id2word, 0,**train_params)
loss = transformer._loss
elif model_name=='bert':
#bert_config = modeling.BertConfig.from_json_file('bert/bert_large/bert_config.json')
bert_large_config_path = 'bert/pre-trained/large/cased_L-24_H-1024_A-16/bert_config.json'
bert_config = modeling.BertConfig.from_json_file(bert_large_config_path)
model = new_model_fn_builder(bert_config)
features = {}
features['input_ids']= tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['input_mask'] = tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['segment_ids']=tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['start_positions'] = tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,)),tf.int32)
features['end_positions'] =tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,)),tf.int32)
loss = model(features)
elif model_name == 'small':
slim = tf.contrib.slim
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
v= tf.get_variable(name='large_variable',shape=(3000,224, 224, 3),trainable=True)
x = tf.slice(v,[0,0,0,0],tf.shape(x),name='large_slice')
net = slim.max_pool2d(x, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.flatten(net)
net = slim.fully_connected(net, 1024, activation_fn=tf.nn.sigmoid,trainable=False)
net = slim.fully_connected(net, 1000, activation_fn=None,trainable=False)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=net)
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9, beta2=0.98, epsilon=1e-9).minimize(
tf.reduce_sum(loss))
# TODO: Make lr, beta, epsilon value of parameter
"""
if opt == 'Adam':
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9, beta2=0.98, epsilon=1e-9).minimize(
tf.reduce_sum(loss))
elif opt == 'GradientDescent':
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=0.2).minimize(tf.reduce_sum(loss))
"""
return optimizer