def test_graph_replace_gradients(self):
ops.reset_default_graph()
w = variables.VariableV1(0.0, name="w")
y = math_ops.multiply(math_ops.multiply(w, w, name="mul1"), w, name="mul2")
g = gradients_impl.gradients(y, w, name="grad")[0]
# Extract the operations.
replacement_ts = {w.value(): g}
original_mul1_grad = (ops.get_default_graph().
get_operation_by_name("grad/mul1_grad/Mul_1"))
# Should not raise exception.
res = ge.graph_replace(g, replacement_ts, dst_scope="res")
# Extract the operations after graph_replace.
result_mul1_grad = (ops.get_default_graph().
get_operation_by_name("res/grad/mul1_grad/Mul_1"))
# Make sure _original_ops are as expected.
self.assertEqual(original_mul1_grad._original_op.name, u"mul1")
self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1")
self.assertNotEqual(res.name, g.name)
with session.Session() as sess:
sess.run(variables.global_variables_initializer())
g_val, res_val = sess.run([g, res])
self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
def create_pgd_graph(lb, ub, sess, tf_input, tf_output, target):
# Replace graph
tf_image = tf.Variable(lb, trainable=True)
tf_output = ge.graph_replace(tf_output, {tf_input: tf_image + 0.0})
# Output diversification
tf_dir = tf.placeholder(shape=(tf_output.shape[1]), dtype=tf.float64)
tf_eps_init = tf.placeholder(shape=lb.shape, dtype=tf.float64)
tf_init_error = tf.reduce_sum(tf_dir * tf_output)
tf_init_grad = tf.gradients(tf_init_error, [tf_image])[0]
tf_train_init = tf_image + tf_eps_init * tf.sign(tf_init_grad)
tf_train_init = tf.assign(tf_image, tf_train_init)
# PGD
tf_train_error = tf.keras.utils.to_categorical(
target, num_classes=tf_output.shape[-1])
tf_eps_pgd = tf.placeholder(shape=lb.shape, dtype=tf.float64)
tf_train_error = tf.keras.losses.categorical_crossentropy(tf_train_error,
tf_output,
from_logits=True)
tf_train_grad = tf.gradients(tf_train_error, [tf_image])[0]
tf_train_pgd = tf_image - tf_eps_pgd * tf.sign(tf_train_grad)
tf_train_pgd = tf.assign(tf_image, tf_train_pgd)
# Clip
tf_train_clip = tf.clip_by_value(tf_image, lb, ub)
tf_train_clip = tf.assign(tf_image, tf_train_clip)
# Seed
tf_seed_pl = tf.placeholder(shape=lb.shape, dtype=tf.float64)
tf_seed = tf.assign(tf_image, tf_seed_pl)
return tf_image, tf_dir, tf_seed_pl, tf_eps_init, tf_eps_pgd, tf_output, tf_train_init, tf_train_pgd, tf_train_clip, tf_seed
def test_graph_replace_gradients(self):
ops.reset_default_graph()
w = variables.Variable(0.0, name="w")
y = math_ops.multiply(math_ops.multiply(w, w, name="mul1"),
w,
name="mul2")
g = gradients_impl.gradients(y, w, name="grad")[0]
# Extract the operations.
replacement_ts = {w.value(): g}
original_mul1_grad = (ops.get_default_graph().get_operation_by_name(
"grad/mul1_grad/Mul_1"))
# Should not raise exception.
res = ge.graph_replace(g, replacement_ts, dst_scope="res")
# Extract the operations after graph_replace.
result_mul1_grad = (ops.get_default_graph().get_operation_by_name(
"res/grad/mul1_grad/Mul_1"))
# Make sure _original_ops are as expected.
self.assertEquals(original_mul1_grad._original_op.name, u"mul1")
self.assertEquals(result_mul1_grad._original_op.name, u"res/mul1")
self.assertNotEquals(res.name, g.name)
with session.Session() as sess:
sess.run(variables.global_variables_initializer())
g_val, res_val = sess.run([g, res])
self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
def graph_replace(*args, **kwargs):
"""
Monkey patch graph_replace so that it works with TF 1.0+
See https://github.com/tensorflow/tensorflow/issues/9978 or https://github.com/tensorflow/tensorflow/issues/9125
"""
for op in tf.get_default_graph().get_operations():
op._original_op = None
return ge.graph_replace(*args, **kwargs)
def test_graph_replace_ordered_dict(self):
ops.reset_default_graph()
a = constant_op.constant(1.0, name="a")
b = variables.Variable(1.0, name="b")
eps = constant_op.constant(0.001, name="eps")
c = array_ops.identity(a + b + eps, name="c")
a_new = constant_op.constant(2.0, name="a_new")
c_new = ge.graph_replace(collections.OrderedDict({"c": c}), {a: a_new})
self.assertTrue(isinstance(c_new, collections.OrderedDict))
def _recompute_gradients(self, state):
"""recomputes gradient of loss at current example and previous iterate."""
replace_dict = self._make_replace_dict(state, self.grads, self.vars)
recomputed_grads = contrib_graph_editor.graph_replace(
self.grads, replace_dict)
return recomputed_grads
def test_graph_replace_ordered_dict(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
c_new = ge.graph_replace(collections.OrderedDict({"c": c}), {a: a_new})
self.assertTrue(isinstance(c_new, collections.OrderedDict))
def test_graph_replace_missing(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.constant(2.0, name="b")
c = a + 2 * b
d = tf.constant(2.0, name="d")
res = ge.graph_replace([b, c], {a: d})
self.assertEqual(res[0].name, "b:0")
self.assertEqual(res[1].name, "add_1:0")
def test_graph_replace_missing(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.constant(2.0, name="b")
c = a + 2 * b
d = tf.constant(2.0, name="d")
res = ge.graph_replace([b, c], {a: d})
self.assertEqual(res[0].name, "b:0")
self.assertEqual(res[1].name, "add_1:0")
def test_graph_replace_named_tuple(self):
ops.reset_default_graph()
a = constant_op.constant(1.0, name="a")
b = variables.Variable(1.0, name="b")
eps = constant_op.constant(0.001, name="eps")
c = array_ops.identity(a + b + eps, name="c")
a_new = constant_op.constant(2.0, name="a_new")
one_tensor = collections.namedtuple("OneTensor", ["t"])
c_new = ge.graph_replace(one_tensor(c), {a: a_new})
self.assertTrue(isinstance(c_new, one_tensor))
def test_graph_replace_named_tuple(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
one_tensor = collections.namedtuple("OneTensor", ["t"])
c_new = ge.graph_replace(one_tensor(c), {a: a_new})
self.assertTrue(isinstance(c_new, one_tensor))
def test_graph_replace(self):
ops.reset_default_graph()
a = constant_op.constant(1.0, name="a")
b = variables.Variable(1.0, name="b")
eps = constant_op.constant(0.001, name="eps")
c = array_ops.identity(a + b + eps, name="c")
a_new = constant_op.constant(2.0, name="a_new")
c_new = ge.graph_replace(c, {a: a_new})
with session.Session() as sess:
sess.run(variables.global_variables_initializer())
c_val, c_new_val = sess.run([c, c_new])
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val, 3.001, ERROR_TOLERANCE)
def test_graph_replace(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
c_new = ge.graph_replace(c, {a: a_new})
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
c_val, c_new_val = sess.run([c, c_new])
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val, 3.001, ERROR_TOLERANCE)
def test_graph_replace(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
c_new = ge.graph_replace(c, {a: a_new})
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
c_val, c_new_val = sess.run([c, c_new])
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val, 3.001, ERROR_TOLERANCE)
def test_graph_replace(self):
ops.reset_default_graph()
a = constant_op.constant(1.0, name="a")
b = variables.Variable(1.0, name="b")
eps = constant_op.constant(0.001, name="eps")
c = array_ops.identity(a + b + eps, name="c")
a_new = constant_op.constant(2.0, name="a_new")
c_new = ge.graph_replace(c, {a: a_new})
with session.Session() as sess:
sess.run(variables.global_variables_initializer())
c_val, c_new_val = sess.run([c, c_new])
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val, 3.001, ERROR_TOLERANCE)
def main():
with tf.Session(graph=tf.Graph()) as sess:
tag_name = "serve"
signature_name = "serving_default"
input_model_path = "./savedmodel/1531987620"
output_model_path = "./converted_savedmodel/1531987620"
graph = tf.get_default_graph()
meta_graph_def = tf.saved_model.loader.load(sess, [tag_name],
input_model_path)
signature_def = meta_graph_def.signature_def[signature_name]
old_input_tensor = graph.get_tensor_by_name(
signature_def.inputs.values()[0].name)
print "Old model input: ", old_input_tensor.dtype, old_input_tensor.shape
new_base64_placeholder = tf.placeholder(shape=[None], dtype=tf.string)
new_input_tensor = tf.map_fn(tf.decode_base64, new_base64_placeholder)
print "New model input: ", old_input_tensor.dtype, old_input_tensor.shape
output_keys = [_.name for _ in signature_def.outputs.values()]
old_output_tensors = [graph.get_tensor_by_name(_) for _ in output_keys]
new_output_tensors = graph_editor.graph_replace(
target_ts=old_output_tensors,
replacement_ts={old_input_tensor: new_input_tensor})
new_output_tensor_infos = [
tf.saved_model.utils.build_tensor_info(_) for _ in new_output_tensors
]
new_signature = tf.saved_model.signature_def_utils.build_signature_def(
inputs={
signature_def.inputs.keys()[0]:
tf.saved_model.utils.build_tensor_info(new_base64_placeholder)
},
outputs=dict(zip(output_keys, new_output_tensor_infos)),
method_name=signature_def.method_name)
builder = tf.saved_model.builder.SavedModelBuilder(output_model_path)
builder.add_meta_graph_and_variables(
sess,
[tag_name],
clear_devices=True,
signature_def_map={
signature_name: new_signature,
},
main_op=None, # avoid duplicate
legacy_init_op=None # avoid duplicate
)
builder.save()
def test_graph_replace_dict(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
c_new = ge.graph_replace({"c": c}, {a: a_new})
self.assertTrue(isinstance(c_new, dict))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
c_val, c_new_val = sess.run([c, c_new])
self.assertTrue(isinstance(c_new_val, dict))
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val["c"], 3.001, ERROR_TOLERANCE)
def test_graph_replace_dict(self):
tf.reset_default_graph()
a = tf.constant(1.0, name="a")
b = tf.Variable(1.0, name="b")
eps = tf.constant(0.001, name="eps")
c = tf.identity(a + b + eps, name="c")
a_new = tf.constant(2.0, name="a_new")
c_new = ge.graph_replace({"c": c}, {a: a_new})
self.assertTrue(isinstance(c_new, dict))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
c_val, c_new_val = sess.run([c, c_new])
self.assertTrue(isinstance(c_new_val, dict))
self.assertNear(c_val, 2.001, ERROR_TOLERANCE)
self.assertNear(c_new_val["c"], 3.001, ERROR_TOLERANCE)
def thread( proc_id, i, args, conn ):
import tensorflow as tf
from pgd_div import create_pgd_graph, pgd
from tensorflow.contrib import graph_editor as ge
print( 'Proc', proc_id )
os.sched_setaffinity(0,[proc_id])
model_path = args.model
sess = tf.Session()
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
g = tf.import_graph_def(graph_def, name='')
tf_out = sess.graph.get_operations()[-1].inputs[0]
tf_in_new = tf.placeholder( shape=(784), dtype=tf.float64, name='x' )
tf_in_old = tf.reshape( tf_in_new, (1,1,1,784) )
tf_in_old = tf.cast( tf_in_old, tf.float32 )
tf_in = tf.get_default_graph().get_tensor_by_name( 'input:0' )
tf_output = ge.graph_replace(tf_out, {tf_in: tf_in_old})
tf_output = tf.cast( tf_output, tf.float64 )
pgd_obj = create_pgd_graph( specLB, specUB, sess, tf_in_new, tf_output, i )
for j in range( args.it ):
ex = pgd(sess, specLB, specUB, *pgd_obj, *pgd_args)
status = conn.poll(0.001)
if status:
if conn.recv() == 'kill':
return
if np.argmax( sess.run( tf_output, feed_dict={tf_in_new: ex} )) == i:
conn.send( (i,ex) )
while True:
status = conn.poll(1)
if status:
if conn.recv() == 'kill':
return
conn.send( (i,False) )
while True:
status = conn.poll(1)
if status:
if conn.recv() == 'kill':
return
def mrcnn_graph(self, mode, config):
with tf.device('/cpu:0'):
if mode == "inference":
batch_size = config.BATCH_SIZE_INFERENCE
if mode == "train":
batch_size = config.BATCH_SIZE_TRAIN
input_images = tf.placeholder(tf.float32, shape=[batch_size, 1024, 1024, config.IMAGE_SHAPE[2]], name="input_image")
image_metas = tf.placeholder(tf.float32, shape=[batch_size, config.IMAGE_META_SIZE], name="metas")
anchors = tf.placeholder(tf.float32, (None, 4), name="anchors")
if mode == "train":
input_rpn_gt_matchs = tf.placeholder(tf.int64, (batch_size, None), name="rpn_gt_match")
input_rpn_gt_deltas = tf.placeholder(tf.float32, (batch_size, None, 4), name="gt_deltas")
input_gt_box = tf.placeholder(tf.float32, (batch_size, None, 4), name="input_gt_box")
input_gt_cls_ids = tf.placeholder(tf.int64, (batch_size, None,), name="gt_cls_ids")
input_gt_mask = tf.placeholder(tf.int64, (batch_size, None) + config.MINI_MASK_SIZE, name="input_gt_mask")
# imageをもとにfeature_mapを作る.
C1, C2, C3, C4, C5 = graphs_for_camera.resnet(input_images)
# feature_mapをもとにPを作る.
psize = config.TOP_DOWN_PYRAMID_SIZE
P5 = tf.layers.conv2d(C5, filters=psize, kernel_size=1, strides=(1, 1), name="fpn_c5p5")
p = tf.keras.layers.UpSampling2D(size=(2, 2), name="fpn_P5_Upsampling")(P5)
c = tf.layers.conv2d(C4, filters=psize, kernel_size=1, strides=(1, 1), name="fpn_c4p4")
P4 = tf.add(p, c, "fpn_p4_Add")
p = tf.keras.layers.UpSampling2D(size=(2, 2), name="fpn_P4_Upsampling")(P4)
c = tf.layers.conv2d(C3, filters=psize, kernel_size=1, strides=(1, 1), name="fpn_c3p3")
P3 = tf.add(p, c, "fpn_p3_Add")
p = tf.keras.layers.UpSampling2D(size=(2, 2), name="fpn_P3_Upsampling")(P3)
c = tf.layers.conv2d(C2, filters=psize, kernel_size=1, strides=(1, 1), name="fpn_c2p2")
P2 = tf.add(p, c, "fpn_p2_Add")
# Attach 3x3 conv to all P layers to get the final feature maps.
P2 = tf.layers.conv2d(P2, filters=psize, kernel_size=3, strides=(1, 1), padding="SAME", name="fpn_p2")
P3 = tf.layers.conv2d(P3, filters=psize, kernel_size=3, strides=(1, 1), padding="SAME", name="fpn_p3")
P4 = tf.layers.conv2d(P4, filters=psize, kernel_size=3, strides=(1, 1), padding="SAME", name="fpn_p4")
P5 = tf.layers.conv2d(P5, filters=psize, kernel_size=3, strides=(1, 1), padding="SAME", name="fpn_p5")
# P6 is used for the 5th anchor scale in RPN. Generated by
# subsampling from P5 with stride of 2.
P6 = tf.nn.max_pool(P5, ksize=(1, 1, 1, 1), strides=(1, 2, 2, 1), padding="SAME")
feature_map = [P2, P3, P4, P5, P6]
mrcnn_map = [P2, P3, P4, P5]
rpn_outputs = []
# rpn = build_rpn_model(config.ANCHOR_STRIDE, config.ANCHOR_PER_LOCATION, config.TOP_DOWN_PYRAMID_SIZE)
input_feature_map, output_rpn = graphs_for_camera.build_rpn_graph(config.ANCHOR_STRIDE, config.ANCHOR_PER_LOCATION, config.TOP_DOWN_PYRAMID_SIZE, config, batch_size)
for P in feature_map:
rpn_output = graph_editor.graph_replace(output_rpn, {input_feature_map: P})
rpn_outputs.append(rpn_output)
# [(cls_logit, cls_prob, deltas), (cls_logit, cls_prob, deltas), (cls_logit, cls_prob, deltas)]
# =>[[cls_logit, cls_logit, ...], [cls_prob, cls_prob, ...], [deltas, deltas, ...]]
rpn_outputs = zip(*rpn_outputs)
# わざわざnameつける必要あるのか.
names = ["rpn_cls_logit", "rpn_cls_prob", "rpn_deltas"]
# (HW2+HW3+HW4+HW5+HW6*A/pix, 2or4)
rpn_cls_logit, rpn_cls_prob, rpn_deltas =\
[tf.concat(output, name=name, axis=1) for output, name in zip(rpn_outputs, names)]
# nmsで対象roisを減らす
rpn_rois, pre_nms_box = graphs_for_camera.proposal_graph(config.POST_NMS_PROPOSALS_INFERENCE,\
config.NMS_THRESHOLD, self.config, batch_size, rpn_cls_prob, rpn_deltas, anchors)
# 以降はboxの処理がroisがベースになる
# rpn_roisの範囲の特徴量マップをRoiAlignで加工し, fpnにかける.
# 各roiのクラス, 枠の補正を出力する.
metas = utils.metas_converter(image_metas)
if mode == "train":
# 学習のtargetを決める
# rpn_roisは条件を満たすものからランダムに選ばれる.
rpn_rois, gt_cls_ids, gt_mrcnn_deltas, gt_mask = utils.batch_slice(
(rpn_rois, input_gt_box, input_gt_cls_ids, input_gt_mask),
self.config.BATCH_SIZE_TRAIN,
lambda x, y, z, w: graphs_for_camera.detection_target_graph(x, y, z, w, config))
#lambda x, y, z, w: graphs_for_camera.detection_target_graph(x, y, z, w, config, positive_indices_master, negative_indices_master))
mrcnn_cls_logit, mrcnn_cls_prob, mrcnn_deltas = graphs_for_camera.fpn_classifer_graph(mrcnn_map, rpn_rois, image_metas,
config.POOL_SIZE, config.NUM_CLASSES, config.FPN_CLASSIFER_FC_FILTER_SIZE, training=False)
mrcnn_mask = graphs_for_camera.fpn_mask_graph(rpn_rois, mrcnn_map, image_metas, config.MASK_POOLSIZE, config.NUM_CLASSES)
# 各anchorに物体が存在するかどうか.
# rpnのlogitとmatchs(0 or 1)を比較.
rpn_cls_loss = graphs_for_camera.rpn_cls_loss_func(rpn_cls_logit, input_rpn_gt_matchs)
# 物体がある項目(match==1)について回帰を行う.
rpn_delta_loss, target_gt_rpn_deltas, target_rpn_deltas = graphs_for_camera.rpn_delta_loss_func(rpn_deltas, input_rpn_gt_deltas, input_rpn_gt_matchs, config)
#
active_ids = utils.metas_converter(image_metas)["activate_class_ids"]
mrcnn_cls_loss = graphs_for_camera.mrcnn_cls_loss_func(mrcnn_cls_logit, gt_cls_ids, active_ids)
mrcnn_delta_loss = graphs_for_camera.mrcnn_delta_loss_func(gt_cls_ids, gt_mrcnn_deltas, mrcnn_deltas)
mrcnn_mask_loss = graphs_for_camera.mrcnn_mask_loss_func(gt_cls_ids, mrcnn_mask, gt_mask)
losses = [rpn_cls_loss, rpn_delta_loss, mrcnn_cls_loss, mrcnn_delta_loss, mrcnn_mask_loss]
inputs = [input_images, image_metas, anchors, input_rpn_gt_deltas, input_rpn_gt_matchs, input_gt_cls_ids, input_gt_box, input_gt_mask]
return inputs, losses
if mode == "inference":
mrcnn_cls_logit, mrcnn_cls_prob, mrcnn_deltas =\
graphs_for_camera.fpn_classifer_graph(mrcnn_map, rpn_rois, image_metas,
config.POOL_SIZE, config.NUM_CLASSES, config.FPN_CLASSIFER_FC_FILTER_SIZE, training=False)
detections = graphs_for_camera.detection_graph(config, rpn_rois, mrcnn_cls_prob, mrcnn_deltas, image_metas)
detection_box, cls_id, score = detections[:, :, :4], detections[:, :, 4], detections[:, :, 5]
mrcnn_mask = graphs_for_camera.fpn_mask_graph(detection_box, mrcnn_map, image_metas, config.MASK_POOLSIZE, config.NUM_CLASSES)
inputs = [input_images, image_metas, anchors]
outputs = [detections, mrcnn_mask, rpn_rois, rpn_deltas]
return inputs, outputs
def _consistency_loss(self):
from tensorflow.contrib.graph_editor import graph_replace
x = self.x_ph
shape_x = mixed_shape(x)
batch_size = shape_x[0]
rand_ids = tf.random_shuffle(tf.range(batch_size, dtype=tf.int32))
x_rand = tf.gather(x, rand_ids)
# (batch_size)
lam = self.beta.sample(batch_size)
lam_x = tf.reshape(lam, [batch_size] + [1] * len(self.x_shape))
lam_y = tf.reshape(lam, [batch_size, 1])
x_mixed = lam_x * x + (1 - lam_x) * x_rand
y_prob_stu = self.get_output('y_dist_stu_sto')['prob']
y_prob_stu_on_x_mixed = graph_replace(y_prob_stu, {x: x_mixed})
if self.cons_against_mean:
print("Use teacher (deterministic) for consistency!")
y_prob_tea = self.get_output('y_dist_tea_det')['prob']
else:
print("Use teacher (stochastic) for consistency!")
assert 'y_dist_tea_sto' in self.output_dict, "'output_dict' must contain 'y_dist_tea_sto'!"
y_prob_tea = self.get_output('y_dist_tea_sto')['prob']
y_prob_tea_rand = graph_replace(y_prob_tea, {x: x_rand})
y_prob_tea_mixed = lam_y * y_prob_tea + (1 - lam_y) * y_prob_tea_rand
if self.cons_mode == 'mse':
# IMPORTANT: Here, we take the sum over classes.
# Implementations from other papers they use 'mean' instead of 'sum'.
# This means our 'cons_coeff' should be about 10 (for CIFAR-10 and SVHN),
# not 100 like in the original papers
print("cons_mode=mse!")
consistency = tf.reduce_sum(
tf.square(y_prob_stu_on_x_mixed -
tf.stop_gradient(y_prob_tea_mixed)),
axis=1)
elif self.cons_mode == 'kld':
print("cons_mode=kld!")
from my_utils.tensorflow_utils.distributions import KLD_2Cats_v2
consistency = KLD_2Cats_v2(y_prob_stu_on_x_mixed,
tf.stop_gradient(y_prob_tea_mixed))
elif self.cons_mode == 'rev_kld':
print("cons_mode=rev_kld!")
from my_utils.tensorflow_utils.distributions import KLD_2Cats_v2
consistency = KLD_2Cats_v2(tf.stop_gradient(y_prob_tea_mixed),
y_prob_stu_on_x_mixed)
else:
raise ValueError("Do not support 'cons_mode'={}!".format(
self.cons_mode))
if self.cons_4_unlabeled_only:
label_flag_inv = self.get_output('label_flag_inv')
num_unlabeled = self.get_output('num_unlabeled')
consistency = tf.reduce_sum(consistency * label_flag_inv,
axis=0) * 1.0 / (num_unlabeled + 1e-8)
else:
consistency = tf.reduce_mean(consistency, axis=0)
results = {
'cons': consistency,
}
return results
def _mur_loss(self):
from tensorflow.contrib.graph_editor import graph_replace
# IMPORTANT: We use 'x_pert_stu' to ensure no perturbation on the input
# (batch, x_dim)
x0 = self.get_output('x_pert_stu')
# IMPORTANT: The output here is 'y_dist_stu_sto' not 'y_dist_stu'
# (batch, num_classes)
y0_prob = self.get_output('y_dist_stu_sto')['prob']
# (batch, )
cond_ent0 = tf.reduce_sum(-y0_prob * log_w_clip(y0_prob), axis=1)
normalized_axes = list(range(1, x0.shape.ndims))
g0 = tf.gradients(cond_ent0, [x0])[0]
g0_norm = tf.stop_gradient(
tf.sqrt(tf.reduce_sum(g0**2, axis=normalized_axes, keepdims=True))
+ 1e-15)
rad = self.mur_noise_radius
# Direct approximation
if self.mur_opt_steps == 1:
print("Direct approximation of x*!")
eps = rad * g0 / (g0_norm + 1e-8)
x_final = tf.stop_gradient(x0 + eps)
else:
lr = self.mur_opt_lr
if self.mur_iter_mode == "grad_asc_w_lagrangian_relax":
print(
"Iterative approximation of x* using vanilla gradient ascent!"
)
x_t = x0
cond_ent_t = cond_ent0
for _ in range(self.mur_opt_steps):
grad_x_t = tf.gradients(cond_ent_t, [x_t])[0]
xt_m_x0_norm = tf.stop_gradient(
tf.sqrt(
tf.reduce_sum((x_t - x0)**2,
axis=normalized_axes,
keepdims=True)) + 1e-15)
# Update 'x_t' and 'cond_ent_t'
x_t = tf.stop_gradient(
x_t + lr *
(grad_x_t - g0_norm / self.mur_noise_radius *
(x_t - x0) * (2 - self.mur_noise_radius /
(xt_m_x0_norm + 1e-15))))
cond_ent_t = graph_replace(cond_ent0,
replacement_ts={x0: x_t})
x_final = x_t
elif self.mur_iter_mode == "proj_grad_asc":
print(
"Iterative approximation of x* using project gradient ascent!"
)
x_t = x0
cond_ent_t = cond_ent0
for _ in range(self.mur_opt_steps):
grad_x_t = tf.gradients(cond_ent_t, [x_t])[0]
z_t = x_t + lr * grad_x_t
# (batch, 1, 1, 1)
zt_m_x0_norm = tf.stop_gradient(
tf.sqrt(
tf.reduce_sum((z_t - x0)**2,
axis=normalized_axes,
keepdims=True)) + 1e-15)
cond = tf.cast(tf.less_equal(zt_m_x0_norm, rad),
dtype=tf.float32)
x_t = cond * z_t + (1.0 -
cond) * (x0 +
(z_t - x0) / zt_m_x0_norm)
x_t = tf.stop_gradient(x_t)
cond_ent_t = graph_replace(cond_ent0,
replacement_ts={x0: x_t})
x_final = x_t
else:
raise ValueError(self.mur_iter_mode)
y_prob_final = graph_replace(y0_prob, replacement_ts={x0: x_final})
if self.mur_mode == "mse_wrt_point":
mur = tf.reduce_sum(
tf.square(tf.stop_gradient(y0_prob) - y_prob_final), axis=1)
elif self.mur_mode == "mse_wrt_neigh":
mur = tf.reduce_sum(tf.square(y0_prob -
tf.stop_gradient(y_prob_final)),
axis=1)
else:
raise ValueError("Do not support mur_mode={}!".format(
self.mur_mode))
if self.mur_4_unlabeled_only:
label_flag_inv = self.get_output('label_flag_inv')
num_unlabeled = self.get_output('num_unlabeled')
mur = tf.reduce_sum(mur * label_flag_inv,
axis=0) * 1.0 / (num_unlabeled + 1e-8)
else:
mur = tf.reduce_mean(mur, axis=0)
return {'grad_norm_avg': tf.reduce_mean(g0_norm), 'mur': mur}
def __init__(self,
structure,
saved_model_dir,
num_sections=19,
*args,
**kwargs):
"""Loads a saved classifier model for the OMR engine.
Args:
structure: A `structure.Structure`.
saved_model_dir: Path to the TF saved_model directory to load.
num_sections: Number of vertical positions of patches to extract, centered
on the middle staff line.
*args: Passed through to `SavedConvolutional1DClassifier`.
**kwargs: Passed through to `SavedConvolutional1DClassifier`.
Raises:
ValueError: If the saved model input could not be interpreted as a 3D
array with the patch size.
"""
super(SavedConvolutional1DClassifier, self).__init__(*args, **kwargs)
sess = tf.get_default_session()
graph_def = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING], saved_model_dir)
signature = None
for key in _SIGNATURE_KEYS:
if key in graph_def.signature_def:
signature = graph_def.signature_def[key]
break
else:
# for/else is only executed if the loop completes without breaking.
raise ValueError(
'One of the following signatures must be present: %s' %
_SIGNATURE_KEYS)
input_info = signature.inputs['input']
if not (len(input_info.tensor_shape.dim) == 3
and input_info.tensor_shape.dim[1].size > 0
and input_info.tensor_shape.dim[2].size > 0):
raise ValueError('Invalid patches input: ' + str(input_info))
patch_height = input_info.tensor_shape.dim[1].size
patch_width = input_info.tensor_shape.dim[2].size
with tf.name_scope('saved_classifier'):
self.staffline_extractor = staffline_extractor.StafflineExtractor(
structure.staff_remover.remove_staves,
structure.staff_detector,
num_sections=num_sections,
target_height=patch_height)
stafflines = self.staffline_extractor.extract_staves()
num_staves = tf.shape(stafflines)[0]
num_sections = tf.shape(stafflines)[1]
staffline_patches = patches.patches_1d(stafflines, patch_width)
staffline_patches_shape = tf.shape(staffline_patches)
patches_per_position = staffline_patches_shape[2]
flat_patches = tf.reshape(staffline_patches, [
num_staves * num_sections * patches_per_position, patch_height,
patch_width
])
# Feed in the flat extracted patches as the classifier input.
predictions_name = signature.outputs[
prediction_keys.PredictionKeys.CLASS_IDS].name
predictions = contrib_graph_editor.graph_replace(
sess.graph.get_tensor_by_name(predictions_name), {
sess.graph.get_tensor_by_name(signature.inputs['input'].name):
flat_patches
})
# Reshape to the original patches shape.
predictions = tf.reshape(predictions, staffline_patches_shape[:3])
# Pad the output. We take only the valid patches, but we want to shift all
# of the predictions so that a patch at index i on the x-axis is centered
# on column i. This determines the x coordinates of the glyphs.
width = tf.shape(stafflines)[-1]
predictions_width = tf.shape(predictions)[-1]
pad_before = (width - predictions_width) // 2
pad_shape_before = tf.concat(
[staffline_patches_shape[:2], [pad_before]], axis=0)
pad_shape_after = tf.concat([
staffline_patches_shape[:2],
[width - predictions_width - pad_before]
],
axis=0)
self.output = tf.concat(
[
# NONE has value 1.
tf.ones(pad_shape_before, tf.int64),
tf.to_int64(predictions),
tf.ones(pad_shape_after, tf.int64),
],
axis=-1)
# run_min_length can be set on the saved model to tweak its behavior, but
# should be overridden by the keyword argument.
if 'run_min_length' not in kwargs:
try:
# Try to read the run min length from the saved model. This is tweaked
# on a per-model basis.
run_min_length_t = sess.graph.get_tensor_by_name(
_RUN_MIN_LENGTH_CONSTANT_NAME)
run_min_length = contrib_util.constant_value(run_min_length_t)
# Implicit comparison is invalid on a NumPy array.
# pylint: disable=g-explicit-bool-comparison
if run_min_length is None or run_min_length.shape != ():
raise ValueError(
'Bad run_min_length: {}'.format(run_min_length))
# Overwrite the property after the Convolutional1DGlyphClassifier
# constructor completes.
self.run_min_length = int(run_min_length)
except KeyError:
pass # No run_min_length tensor in the saved model.
def build_HoloGAN(self):
self.view_in = tf.placeholder(tf.float32, [None, 6], name='view_in')
self.inputs = tf.placeholder(
tf.float32,
[None, self.output_height, self.output_width, self.c_dim],
name='real_images')
self.z = tf.placeholder(tf.float32, [None, cfg.z_dim], name='z')
inputs = self.inputs
gen_func = eval("self." + (cfg.generator))
dis_func = eval("self." + (cfg.discriminator))
self.gen_view_func = eval(cfg.view_func)
self.G = gen_func(self.z, self.view_in)
if str.lower(str(cfg.style_disc)) == "true":
print("Style Disc")
self.D, self.D_logits, _, self.d_h1_r, self.d_h2_r, self.d_h3_r, self.d_h4_r = dis_func(
inputs, cont_dim=cfg.z_dim, reuse=False)
self.D_, self.D_logits_, self.Q_c_given_x, self.d_h1_f, self.d_h2_f, self.d_h3_f, self.d_h4_f = dis_func(
self.G, cont_dim=cfg.z_dim, reuse=True)
self.d_h1_loss = cfg.DStyle_lambda * (
tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h1_r, tf.ones_like(self.d_h1_r))) \
+ tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h1_f, tf.zeros_like(self.d_h1_f))))
self.d_h2_loss = cfg.DStyle_lambda * (
tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h2_r, tf.ones_like(self.d_h2_r))) \
+ tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h2_f, tf.zeros_like(self.d_h2_f))))
self.d_h3_loss = cfg.DStyle_lambda * (
tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h3_r, tf.ones_like(self.d_h3_r))) \
+ tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h3_f, tf.zeros_like(self.d_h3_f))))
self.d_h4_loss = cfg.DStyle_lambda * (
tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h4_r, tf.ones_like(self.d_h4_r))) \
+ tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.d_h4_f, tf.zeros_like(self.d_h4_f))))
else:
self.D, self.D_logits, _ = dis_func(inputs,
cont_dim=cfg.z_dim,
reuse=False)
self.D_, self.D_logits_, self.Q_c_given_x = dis_func(
self.G, cont_dim=cfg.z_dim, reuse=True)
self.d_loss_real = tf.reduce_mean(
sigmoid_cross_entropy_with_logits(self.D_logits,
tf.ones_like(self.D)))
self.d_loss_fake = tf.reduce_mean(
sigmoid_cross_entropy_with_logits(self.D_logits_,
tf.zeros_like(self.D_)))
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss = tf.reduce_mean(
sigmoid_cross_entropy_with_logits(self.D_logits_,
tf.ones_like(self.D_)))
if str.lower(str(cfg.style_disc)) == "true":
print("Style disc")
self.d_loss = self.d_loss + self.d_h1_loss + self.d_h2_loss + self.d_h3_loss + self.d_h4_loss
# ====================================================================================================================
# Identity loss
self.q_loss = cfg.lambda_latent * tf.reduce_mean(
tf.square(self.Q_c_given_x - self.z))
self.d_loss = self.d_loss + self.q_loss
self.g_loss = self.g_loss + self.q_loss
# ====================================================================================================================
# Emb loss
if ADD_EMBEDDING:
self.emb_reconstructed = tf.image.resize_images(
self.G, [cfg.emb_input_size] * 2)
self.emb_reconstructed = tf.transpose(self.emb_reconstructed,
[0, 3, 1, 2])
self.emb_reconstructed = ge.graph_replace(
self.emb_output, {self.emb_input: self.emb_reconstructed})
self.e_loss = cfg.lambda_emb * tf.reduce_mean(
tf.square(self.emb_reconstructed -
self.z[:, cfg.z_dim - cfg.emb_dim:]))
self.g_loss = self.g_loss + self.e_loss
self.d_loss_real_sum = scalar_summary("d_loss_real", self.d_loss_real)
self.d_loss_fake_sum = scalar_summary("d_loss_fake", self.d_loss_fake)
self.g_loss_sum = scalar_summary("g_loss", self.g_loss)
self.d_loss_sum = scalar_summary("d_loss", self.d_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.saver = tf.train_on_loader.Saver()
def vectorize_model(model_vars, *o_outs, augment=0):
"""
Function that "vectorizes" a model (as a computation graph).
Given a model written in a "standard way", i.e. with parameters organized in k-rank tensors as needed
(matrices and vectors for linearities, 3 or 4 rank tensors for convolutional kernels etc.), returns the same model
with all the parameters organized in a single vector.
It is believed that having coded the models in this way, it will be much simpler to implement future algorithms,
especially when relying on the second-order derivatives (Hessian of objective function), since in such a framework,
the model will be entirely dependent by its single parameter vector and described by the resulting computation
graph.
Since the the downward process of vectorizing the model does not work, it is
necessary to modify the computation graph accordingly to have an upward dependence
from the all weights vector to the single parameters.
SOLVED! { The technical caveat is that the variables must be encapsulated into an op (tf.identity) otherwise,
for some
unknown reason, the substitution process is not possible. I've tried in several different ways but they all failed.
Substantially you need to keep track of the tf.identity(variable) and use the resulting tensor to build up the model
and then also of the initial value of variable. Probably it is not necessary to keep track of variable itself. }
:param model_vars: list of variables of the model or initializers
:param o_outs: output_variables, list or tensor. (e.g. model output)
:param augment: (int: default 0) augment the all weights vector by creating augumented variables (initialized at 0)
that mirror rank and dimensions of the variables in `model_vars`. The process is repeated
`augment` times.
This new variables can be accessed with methods in `MergedVariable`.
The common usage is to prepare the model to be optimized with optimizers that require states
such as `MomentumOptimizer` (`augment=1`) or `AdamOptimizer` (`augment=2`).
:return: a list which has as first element the `MergedVariable` that represents the all weights vector. Remaining
elements are the outputs
in the modified graph. These new outs are the same computed by the initial model
(by the computation graph in which the model lives) but with dependencies form the all_weight_vector.
"""
# assert len(model_vars) == len(model_vars_tensor), 'length of model_vars and model_var_tensor do not match'
assert len(model_vars) > 0, 'no variables in model_vars!'
outs = [
tf.identity(o) if isinstance(o, tf.Variable) else o for o in o_outs
]
# TODO implement recursive call for nested lists
with model_vars[0].graph.as_default():
true_w = MergedVariable(model_vars)
if augment:
augmented_variables = [true_w]
for k in range(augment):
with tf.name_scope('augmented_variables'):
with tf.name_scope(str(k)):
with tf.name_scope('original_variables'):
tmp_augmented = [
tf.Variable(tf.zeros(v.get_shape().as_list()),
name=v.name.split(':')[0])
for v in model_vars
]
augmented_variables.append(
MergedVariable(tmp_augmented))
w = MergedVariable(augmented_variables)
else:
w = true_w
new_outs = ge.graph_replace(outs, w.generate_swap_dict(
)) # FIXME deprecation here on GraphKey usage...
return [w] + new_outs
#!/usr/bin/env python
import tensorflow as tf
from tensorflow.contrib import graph_editor as ge
from tvm.contrib import tf_op
graph = tf.Graph()
with graph.as_default():
input_op = tf.constant([1.0, 2.0, -1.0])
tf_addone = tf.add(input_op, 1.0)
output_op = tf_addone * 100
tvm_addone = tf_op.Module("tvm_addone_dll.so")["addone"](input_op)
new_output_op = ge.graph_replace(output_op, {tf_addone: tvm_addone})
with tf.Session(graph=graph) as sess:
print(sess.run(output_op))
print(sess.run(new_output_op))
# 計算グラフを作成
x, y = f()
# x と互換性のあるテンソル
new_x = tf.placeholder(x.dtype, x.get_shape(), name='x_copy')
t = OrderedDict()
t2 = OrderedDict()
t[x] = new_x
t2[x] = y
print(t)
# x の代わりに x_copy を使って、x から y への計算をやりなおす
t = graph_editor.graph_replace(
t2, # 複製元の出力
t, # {複製元の入力: 新しい入力}
)
print(x)
print(y)
print(new_x)
print(t)
# graph_replace will keep the shape of target_ts.
# replacement_ts should be a dict, contains replacement_ts[origin(key)] = replacement(value)
# If target_ts is a tensor, then graph_replace will return a tensor
# But if target_ts is a dict e.g. in unrolled target_ts[key] = y,
# Then the return will keep the key and the structure of target_ts, returning:
# target_ts[key] = replaced_y, with replaced_y calculated from replacement_ts.
# holy crap
