def bert(bsize=None):
import sys
sys.path.insert(0, './bert/')
from bert.runsquad import new_model_fn_builder
import modeling
bert_config = modeling.BertConfig.from_json_file("bert/bert_large/bert_config.json")
model = new_model_fn_builder(bert_config)
features = {}
features["input_ids"]= tf.cast(100*tf.placeholder(tf.float32,shape=(bsize,128)),tf.int32)
features["input_mask"] = tf.cast(100*tf.placeholder(tf.float32,shape=(bsize,128)),tf.int32)
features["segment_ids"]=tf.cast(100*tf.placeholder(tf.float32,shape=(bsize,128)),tf.int32)
features["start_positions"] = tf.cast(100*tf.placeholder(tf.float32,shape=(bsize,)),tf.int32)
features["end_positions"] =tf.cast(100*tf.placeholder(tf.float32,shape=(bsize,)),tf.int32)
loss = model(features)
optimizer = tf.train.AdamOptimizer(learning_rate=0.2).minimize(tf.reduce_sum(loss))
return optimizer
def model_fn(scope, batch_size):
from bert.runsquad import new_model_fn_builder
import modeling
bert_config = modeling.BertConfig.from_json_file(
"bert/bert_large/bert_config.json")
model = new_model_fn_builder(bert_config)
features = {}
with tf.variable_scope(scope):
features["input_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["input_mask"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["segment_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["start_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
features["end_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
loss = model(features)
return loss
def model_fn(model_name, batch_size):
if model_name == "vgg19":
from tensorflow.contrib.slim.nets import vgg
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = vgg.vgg_19(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet200":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_200(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet101":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_101(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "resnet152":
from tensorflow.contrib.slim.nets import resnet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
output, _ = resnet_v2.resnet_v2_152(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "nasnet_cifar":
from tensorflow.contrib.slim.nets import nasnet
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = nasnet.build_nasnet_cifar(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "mobile_net":
from tensorflow.contrib.slim.nets import mobilenet_v2
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = mobilenet_v2.mobilenet(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "inceptionv3":
from tensorflow.contrib.slim.nets import inception_v3
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = inception_v3.inception_v3(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name == "transformer":
import modeltransformer.transformer as transf
from modeltransformer.data import DatasetManager
dm = DatasetManager("wmt14")
dm.maybe_download_data_files()
dm.load_vocab()
transformer = transf.Transformer(
num_heads=8,
d_model=512,
d_ff=2048,
model_name=model_name,
tf_sess_config=dict(allow_soft_placement=True))
train_params = dict(
learning_rate=1e-4,
batch_size=batch_size,
seq_len=10,
max_steps=300000,
)
transformer.build_model("wmt14", dm.source_id2word, dm.target_id2word,
0, **train_params)
loss = transformer._loss
elif model_name == "bert":
from bert.runsquad import new_model_fn_builder
import modeling
bert_config = modeling.BertConfig.from_json_file(
"bert/bert_large/bert_config.json")
model = new_model_fn_builder(bert_config)
features = {}
features["input_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["input_mask"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["segment_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 128)),
tf.int32)
features["start_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
features["end_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )), tf.int32)
loss = model(features)
elif model_name == "small":
slim = tf.contrib.slim
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
v = tf.get_variable(name="large_variable",
shape=(3000, 224, 224, 3),
trainable=True)
x = tf.slice(v, [0, 0, 0, 0], tf.shape(x), name="large_slice")
net = slim.max_pool2d(x, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5], trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.flatten(net)
net = slim.fully_connected(net,
1024,
activation_fn=tf.nn.sigmoid,
trainable=False)
net = slim.fully_connected(net,
1000,
activation_fn=None,
trainable=False)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=net)
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9,
beta2=0.98,
epsilon=1e-9).minimize(
tf.reduce_sum(loss))
return optimizer
def model_fn(batch_size, model_name):
if model_name == "bert":
from bert.runsquad import new_model_fn_builder
import modeling
bert_config = modeling.BertConfig.from_json_file(
"bert/bert_large/bert_config.json")
model = new_model_fn_builder(bert_config)
features = {}
if True:
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
features["input_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.int32)
features["input_mask"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.int32)
features["segment_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.int32)
features["start_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.int32)
features["end_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.int32)
loss, layer_outputs, layer_scopes = model(features)
return loss, [features["input_ids"]
] + layer_outputs, ["input"] + layer_scopes
elif model_name == "xl_net":
import xl_net.run_squad as rsq
model = rsq.get_model_fn()
features = {}
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
features["input_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.int32)
features["input_mask"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.float32)
features["segment_ids"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 384)),
tf.int32)
features["start_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.int32)
features["end_positions"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.int32)
features["cls_index"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.int32)
features["is_impossible"] = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, )),
tf.float32)
loss, layer_outputs, layer_scopes = model(features)
return loss, [features["input_ids"]] + layer_outputs, ["input"
] + layer_scopes
elif model_name == "vgg_19":
import vgg
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
loss, endpoints, scopes = vgg.vgg_19(x, y, 1000)
return loss, [x] + endpoints, ["input"] + scopes
elif model_name == "resnet152":
import resnet152_v2
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
loss, endpoints, scopes = resnet152_v2.resnet_v2_152(x, y, 1000)
return loss, [x] + endpoints, ["input"] + scopes
elif model_name == "resnet50":
import resnet50_v2
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1, 1, 1000))
loss, endpoints, scopes = resnet50_v2.resnet_v2_50(x, y, 1000)
return loss, [x] + endpoints, ["input"] + scopes
elif model_name == "inception_v3":
import inception_v3
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
loss, endpoints, scopes = inception_v3.inception_v3(x, y, 1000)
return loss, [x] + endpoints, ["input"] + scopes
elif model_name == "transformer":
import transformer
with tf.variable_scope("input", reuse=tf.AUTO_REUSE):
x = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 100)),
tf.int32)
decode_input = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 100)),
tf.int32)
y = tf.cast(
100 * tf.placeholder(tf.float32, shape=(batch_size, 100)),
tf.int32)
loss, endpoints, scopes = transformer.Transformer().train(
x, decode_input, y)
return loss, [x] + endpoints, ["input"] + scopes
def model_fn(model_name, batch_size):
if model_name=='vgg19':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = vgg.vgg_19(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet200':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_200(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet101':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_101(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='resnet152':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1,1, 1000))
output, _ = resnet_v2.resnet_v2_152(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='nasnet_cifar':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = nasnet.build_nasnet_cifar(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='mobile_net':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size,1000))
output, _ = mobilenet_v2.mobilenet(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='inceptionv3':
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
output, _ = inception.inception_v3(x, 1000)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=output)
elif model_name=='transformer':
dm = DatasetManager('wmt14')
dm.maybe_download_data_files()
dm.load_vocab()
transformer = transf.Transformer(
num_heads=8,
d_model=512,
d_ff=2048,
model_name=model_name,
tf_sess_config=dict(allow_soft_placement=True)
)
train_params = dict(
learning_rate=1e-4,
batch_size=batch_size,
seq_len=10,
max_steps=300000,
)
transformer.build_model('wmt14', dm.source_id2word, dm.target_id2word, 0,**train_params)
loss = transformer._loss
elif model_name=='bert':
#bert_config = modeling.BertConfig.from_json_file('bert/bert_large/bert_config.json')
bert_large_config_path = 'bert/pre-trained/large/cased_L-24_H-1024_A-16/bert_config.json'
bert_config = modeling.BertConfig.from_json_file(bert_large_config_path)
model = new_model_fn_builder(bert_config)
features = {}
features['input_ids']= tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['input_mask'] = tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['segment_ids']=tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,128)),tf.int32)
features['start_positions'] = tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,)),tf.int32)
features['end_positions'] =tf.cast(100*tf.placeholder(tf.float32,shape=(batch_size,)),tf.int32)
loss = model(features)
elif model_name == 'small':
slim = tf.contrib.slim
x = tf.placeholder(tf.float32, shape=(batch_size, 224, 224, 3))
y = tf.placeholder(tf.float32, shape=(batch_size, 1000))
v= tf.get_variable(name='large_variable',shape=(3000,224, 224, 3),trainable=True)
x = tf.slice(v,[0,0,0,0],tf.shape(x),name='large_slice')
net = slim.max_pool2d(x, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.conv2d(net, 128, [5, 5],trainable=False)
net = slim.max_pool2d(net, [2, 2], 2)
net = slim.flatten(net)
net = slim.fully_connected(net, 1024, activation_fn=tf.nn.sigmoid,trainable=False)
net = slim.fully_connected(net, 1000, activation_fn=None,trainable=False)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=net)
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9, beta2=0.98, epsilon=1e-9).minimize(
tf.reduce_sum(loss))
# TODO: Make lr, beta, epsilon value of parameter
"""
if opt == 'Adam':
optimizer = tf.train.AdamOptimizer(learning_rate=0.2,
beta1=0.9, beta2=0.98, epsilon=1e-9).minimize(
tf.reduce_sum(loss))
elif opt == 'GradientDescent':
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=0.2).minimize(tf.reduce_sum(loss))
"""
return optimizer