def test_chain_rewrite(linearize=False):
"""Take chain of length 5, save 2 nodes, make sure 2 units of RAM is
saved."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 5
a0, a1, a2, a3, a4 = make_chain_tanh(n)
grad = memory_saving_gradients.gradients([a4], [a0], checkpoints=[a1,a3])[0]
expected_peak = (n+1-2)*10**6 # subtract 2 since we recompute 2
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1e6+10000, "Difference too large."
def test_resnet():
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
with tf.control_dependencies([a]):
grad = tf.gradients([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
linearize_lib.linearize(grad)
sessrun(grad.op)
peak_memory = cpu_peak()
# 1 for activation of each tanh node + 1 for initial backprop node
# + 1 temporary memory for computing the adds
expected_peak = (n)*10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 10000, "Difference too large."
def test_chain_memory(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="memory" strat."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # for n=5, only choice of a2 saves memory, and alg picks a3
# hence use n>5 to avoid this edge condition
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
expected_peak = (n+1-1)*10**6 # 1 for each node + 1 for generated - 1 saved
# "loss" tensor
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 10000, "Difference too large."
def test_chain_rewrite(linearize=False):
"""Take chain of length 5, save 2 nodes, make sure 2 units of RAM is
saved."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 5
a0, a1, a2, a3, a4 = make_chain_tanh(n)
grad = memory_saving_gradients.gradients([a4], [a0], checkpoints=[a1,
a3])[0]
expected_peak = (n + 1 - 2) * 10**6 # subtract 2 since we recompute 2
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory -
expected_peak) < 1e6 + 10000, "Difference too large."
def convexify_constr(constr):
"""
:param constr: a constraint of a problem
:return:
for a dcp constraint, return itself;
for a non-dcp constraint, return a convexified constraint and domain constraints;
return None if non-sub/super-diff
"""
if not constr.is_dcp():
dom = []
# left hand concave
if constr.args[0].curvature == 'CONCAVE':
left = linearize(constr.args[0])
if left is None:
return None
else:
for con in constr.args[0].domain:
dom.append(con)
else:
left = constr.args[0]
#right hand convex
if constr.args[1].curvature == 'CONVEX':
right = linearize(constr.args[1])
if right is None:
return None
else:
for con in constr.args[1].domain:
dom.append(con)
else:
right = constr.args[1]
return left <= right, dom
else:
return constr
def test_chain_tarjan(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="tarjan"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # for n=5, only choice of a2 saves memory, and alg picks a3
# hence use n>5 to avoid this edge condition
nodes = util.make_chain_tanh_fill(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
expected_peak = 5e6 # originally needed 7 units, now a3,a5 are recomputed
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1e5, "Difference too large."
def test_chain_memory(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="memory" strat."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # for n=5, only choice of a2 saves memory, and alg picks a3
# hence use n>5 to avoid this edge condition
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
expected_peak = (n + 1 -
1) * 10**6 # 1 for each node + 1 for generated - 1 saved
# "loss" tensor
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 10000, "Difference too large."
def test_resnet():
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
with tf.control_dependencies([a]):
grad = tf.gradients([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
linearize_lib.linearize(grad)
sessrun(grad.op)
peak_memory = cpu_peak()
# 1 for activation of each tanh node + 1 for initial backprop node
# + 1 temporary memory for computing the adds
expected_peak = (n) * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 10000, "Difference too large."
def convexify_constr(constr):
"""
:param constr: a constraint of a problem
:return:
for a dcp constraint, return itself;
for a non-dcp constraint, return a convexified constraint and domain constraints;
return None if non-sub/super-diff
"""
if not constr.is_dcp():
dom = []
# left hand concave
if constr.args[0].curvature == 'CONCAVE':
left = linearize(constr.args[0])
if left is None:
return None
else:
for con in constr.args[0].domain:
dom.append(con)
else:
left = constr.args[0]
#right hand convex
if constr.args[1].curvature == 'CONVEX':
right = linearize(constr.args[1])
if right is None:
return None
else:
for con in constr.args[1].domain:
dom.append(con)
else:
right = constr.args[1]
return left<=right, dom
else:
return constr
def test_variables(): tf.reset_default_graph() a = tf.Variable(1.) b = tf.square(a) c = tf.tanh(b) linearize_lib.linearize(c) assert b.op.control_inputs == [] # no control dependency on var initializer
def test_chain_tarjan(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="tarjan"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # for n=5, only choice of a2 saves memory, and alg picks a3
# hence use n>5 to avoid this edge condition
nodes = util.make_chain_tanh_fill(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
linearize_lib.linearize()
peak_memory = cpu_peak()
expected_peak = 5e6 # originally needed 7 units, now a3,a5 are recomputed
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1e5, "Difference too large."
def test_dependent_targets():
tf.reset_default_graph()
a = tf.constant([1], name='a')
c = tf.constant([4], name='c')
result = tf.nn.top_k(a, name='result')
b = tf.add_n([result[0], result[1], c], name='b')
d = tf.constant([7], name='d')
linearize_lib.linearize([b, c])
sess = _create_session()
assert list(sess.run(b)) == [5]
def test_dependent_targets(): tf.reset_default_graph() a = tf.constant([1], name='a') c = tf.constant([4], name='c') result = tf.nn.top_k(a, name='result') b = tf.add_n([result[0],result[1],c],name='b') d = tf.constant([7], name='d') linearize_lib.linearize([b, c]) sess = _create_session() assert list(sess.run(b)) == [5]
def test_dependent_targets_easy(): tf.reset_default_graph() a = tf.constant([1,2,3], name='a') c = tf.constant([4,5,6], name='c') result = tf.nn.top_k(a, name='result') b = result[0]+result[1]+c d = tf.constant([7,8,9]) linearize_lib.linearize([b, c]) sess = _create_session() assert list(sess.run(b)) == [9, 10, 11]
def test_dependent_targets_easy():
tf.reset_default_graph()
a = tf.constant([1, 2, 3], name='a')
c = tf.constant([4, 5, 6], name='c')
result = tf.nn.top_k(a, name='result')
b = result[0] + result[1] + c
d = tf.constant([7, 8, 9])
linearize_lib.linearize([b, c])
sess = _create_session()
assert list(sess.run(b)) == [9, 10, 11]
def test_linearize():
nodes = make_caterpillar_graph(5)
linearize.linearize()
sess = create_session()
import memory_util
memory_util.vlog(1)
with memory_util.capture_stderr() as stderr:
sess.run(nodes[-1].op)
memory_util.print_memory_timeline(stderr, ignore_less_than_bytes=1000)
def test_linearize():
nodes = make_caterpillar_graph(5)
linearize.linearize()
sess = create_session()
import memory_util
memory_util.vlog(1)
with memory_util.capture_stderr() as stderr:
sess.run(nodes[-1].op)
memory_util.print_memory_timeline(stderr, ignore_less_than_bytes=1000)
def test_cifar_resnet_grads():
tf.reset_default_graph()
tf.set_random_seed(1)
ctx = tf.device("/cpu:0")
ctx.__enter__()
loss = _create_cifar_resnet_loss()
grads = tf.gradients(loss, tf.trainable_variables())
linearize_lib.linearize(grads)
sess = _create_session()
sess.run(tf.global_variables_initializer())
grads0 = sess.run(grads)
def test_cifar_resnet_grads():
tf.reset_default_graph()
tf.set_random_seed(1)
ctx = tf.device("/cpu:0")
ctx.__enter__()
loss = _create_cifar_resnet_loss()
grads = tf.gradients(loss, tf.trainable_variables())
linearize_lib.linearize(grads)
sess = _create_session()
sess.run(tf.global_variables_initializer())
grads0 = sess.run(grads)
def test_imagenet_resnet_grads():
tf.reset_default_graph()
tf.set_random_seed(1)
loss = _create_imagenet_resnet_loss()
sess = _create_session()
sess.run(tf.global_variables_initializer())
loss0 = sess.run(loss)
grads = tf.gradients(loss, tf.trainable_variables())
linearize_lib.linearize(grads)
grads0 = sess.run(grads)
print(grads0[0][0, 0, 0, 0]) # -0.00288249
expected_loss0 = 3423.3474
assert abs(loss0 - expected_loss0) < 1e-3
def test_imagenet_resnet_grads(): tf.reset_default_graph() tf.set_random_seed(1) loss = _create_imagenet_resnet_loss() sess = _create_session() sess.run(tf.global_variables_initializer()) loss0 = sess.run(loss) grads = tf.gradients(loss, tf.trainable_variables()) linearize_lib.linearize(grads) grads0 = sess.run(grads) print(grads0[0][0,0,0,0]) # -0.00288249 expected_loss0 = 3423.3474 assert abs(loss0-expected_loss0)<1e-3
def test_cifar_resnet_loss():
tf.reset_default_graph()
tf.set_random_seed(1)
ctx = tf.device("/cpu:0")
ctx.__enter__()
loss = _create_cifar_resnet_loss()
linearize_lib.linearize(loss)
sess = _create_session()
sess.run(tf.global_variables_initializer())
loss0 = sess.run(loss)
expected_loss0 = 20 # 9.0879955 #12.3753
print(expected_loss0)
assert loss0-expected_loss0<1e-3
def test_cifar_resnet_loss():
tf.reset_default_graph()
tf.set_random_seed(1)
ctx = tf.device("/cpu:0")
ctx.__enter__()
loss = _create_cifar_resnet_loss()
linearize_lib.linearize(loss)
sess = _create_session()
sess.run(tf.global_variables_initializer())
loss0 = sess.run(loss)
expected_loss0 = 20 # 9.0879955 #12.3753
print(expected_loss0)
assert loss0 - expected_loss0 < 1e-3
def test_long_chain_tarjan(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="tarjan"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
added = linearize_lib.linearize()
peak_memory = cpu_peak()
# points picked
# a09:0,19:0,a29:0,a39:0,a49:0,a58:0,a68:0,a78:0,a88:0,a97:0
expected_peak = 18e6
util.report_memory(peak_memory, expected_peak)
# todo: remove "REMOVE_ASSERTS"
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_long_chain_memory(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="memory"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
tf.add_to_collection("checkpoints", nodes[10])
tf.add_to_collection("checkpoints", nodes[20])
#grad = memory_saving_gradients.gradients_collection([a], [a0])[0]
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
added = linearize_lib.linearize()
peak_memory = cpu_peak()
# 20 mem used with following tensors picked automatically as bottlenecks
# ['a10:0', 'a19:0', 'a28:0', 'a37:0', 'a46:0', 'a55:0', 'a64:0', 'a73:0',
# 'a82:0', 'a91:0']
expected_peak = 20 * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_resnet_rewrite_memory(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # use n>5 (see test_chain_memory)
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
checkpoints = [nodes[3], nodes[5]] # ['a03_add:0', 'a05_add:0']
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
# 1 for activation of each tanh node + 1 for initial backprop node
# + 1 temporary memory for computing the adds,
# -1 for discarding, then recomputing a1_tanh
expected_peak = (n + 1 + 1 - 1) * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory -
expected_peak) < 1.1 * 10**6, "Difference too large."
def test_resnet_rewrite_tarjan(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # use n>5 (see test_chain_memory)
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
checkpoints = [nodes[3], nodes[5]] # ['a03_add:0', 'a05_add:0']
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
expected_peak = 4e6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory -
expected_peak) < 1.1 * 10**6, "Difference too large."
def test_resnet_rewrite_tarjan(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # use n>5 (see test_chain_memory)
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
checkpoints = [nodes[3], nodes[5]] # ['a03_add:0', 'a05_add:0']
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
expected_peak = 4e6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1*10**6, "Difference too large."
def test_long_resnet_rewrite_tarjan(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
start_time = time.time()
with tf.control_dependencies([a]):
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
start_time = time.time()
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
# 20 mem used with following tensors picked automatically
# ['a10_add:0', 'a19_add:0', 'a28_add:0', 'a37_add:0', 'a46_add:0',
# 'a55_add:0', 'a64_add:0', 'a73_add:0', 'a82_add:0', 'a91_add:0']
expected_peak = 18 * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_resnet_rewrite_memory(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 6 # use n>5 (see test_chain_memory)
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
checkpoints = [nodes[3], nodes[5]] # ['a03_add:0', 'a05_add:0']
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
# 1 for activation of each tanh node + 1 for initial backprop node
# + 1 temporary memory for computing the adds,
# -1 for discarding, then recomputing a1_tanh
expected_peak = (n+1+1-1)*10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1*10**6, "Difference too large."
def test_long_resnet_rewrite_tarjan(linearize=False):
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_resnet(n)
a0 = nodes[0]
a = nodes[-1]
start_time = time.time()
with tf.control_dependencies([a]):
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
start_time = time.time()
if linearize:
added = linearize_lib.linearize(grad.op)
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
peak_memory = cpu_peak()
# 20 mem used with following tensors picked automatically
# ['a10_add:0', 'a19_add:0', 'a28_add:0', 'a37_add:0', 'a46_add:0',
# 'a55_add:0', 'a64_add:0', 'a73_add:0', 'a82_add:0', 'a91_add:0']
expected_peak = 18 * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_long_chain_tarjan(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="tarjan"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
grad = memory_saving_gradients.gradients_tarjan([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
added = linearize_lib.linearize()
peak_memory = cpu_peak()
# points picked
# a09:0,19:0,a29:0,a39:0,a49:0,a58:0,a68:0,a78:0,a88:0,a97:0
expected_peak = 18e6
util.report_memory(peak_memory, expected_peak)
# todo: remove "REMOVE_ASSERTS"
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_long_chain_memory(linearize=False):
"""Like test_chain, but use automatic rewriting with checkpoints="memory"
strategy."""
tf.reset_default_graph()
tf_dev = tf.device('/cpu:0')
tf_dev.__enter__()
n = 100
nodes = make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
tf.add_to_collection("checkpoints", nodes[10])
tf.add_to_collection("checkpoints", nodes[20])
#grad = memory_saving_gradients.gradients_collection([a], [a0])[0]
grad = memory_saving_gradients.gradients_memory([a], [a0])[0]
sess = create_session()
sessrun(tf.global_variables_initializer())
sessrun(grad.op)
if linearize:
added = linearize_lib.linearize()
peak_memory = cpu_peak()
# 20 mem used with following tensors picked automatically as bottlenecks
# ['a10:0', 'a19:0', 'a28:0', 'a37:0', 'a46:0', 'a55:0', 'a64:0', 'a73:0',
# 'a82:0', 'a91:0']
expected_peak = 20 * 10**6
util.report_memory(peak_memory, expected_peak)
if not REMOVE_ASSERTS:
assert (peak_memory - expected_peak) < 1.1e6, "Difference too large."
def test_targets():
tf.reset_default_graph()
n = 5
g = tf.get_default_graph()
nodes1 = util.make_chain_tanh_constant(n, "a")
nodes2 = util.make_chain_tanh_constant(n, "b")
a0,b0 = nodes1[0], nodes2[0]
a, b = nodes1[-1], nodes2[-1]
grad1 = tf.gradients([a], [a0, b0])
grad2 = tf.gradients([b], [a0, b0])
assert linearize_lib.linearize(grad1) == 3
old_version = g._version
assert linearize_lib.linearize(grad1) == 0
assert g._version == old_version
assert linearize_lib.linearize(grad2) == 3
assert linearize_lib.linearize(grad2) == 0
def test_targets():
tf.reset_default_graph()
n = 5
g = tf.get_default_graph()
nodes1 = util.make_chain_tanh_constant(n, "a")
nodes2 = util.make_chain_tanh_constant(n, "b")
a0, b0 = nodes1[0], nodes2[0]
a, b = nodes1[-1], nodes2[-1]
grad1 = tf.gradients([a], [a0, b0])
grad2 = tf.gradients([b], [a0, b0])
assert linearize_lib.linearize(grad1) == 3
old_version = g._version
assert linearize_lib.linearize(grad1) == 0
assert g._version == old_version
assert linearize_lib.linearize(grad2) == 3
assert linearize_lib.linearize(grad2) == 0
def test_chain_linearize():
tf.reset_default_graph()
n = 5
nodes = util.make_chain_tanh_constant(n)
a0 = nodes[0]
a = nodes[-1]
order1 = linearize_lib.obtain_linear_order()
observed_order1 = [n.name for n in order1]
num_new_deps = linearize_lib.linearize()
assert num_new_deps == 0
def test_golden_order(): tf.reset_default_graph() n = 5 nodes = util.make_chain_tanh(n) a0 = nodes[0] a = nodes[-1] grad = tf.gradients([a], [a0])[0] order = linearize_lib.linearize(modify_graph=False) golden_order = ['a00/read', 'a01', 'a02', 'a03', 'gradients/Shape', 'gradients/grad_ys_0', 'gradients/Fill', 'a04', 'gradients/a04_grad/TanhGrad', 'gradients/a03_grad/TanhGrad', 'gradients/a02_grad/TanhGrad', 'gradients/a01_grad/TanhGrad', 'ones'] observed_order = [n.name for n in order] assert observed_order == golden_order
def test_caterpillar_linearize(): tf.reset_default_graph() n = 5 nodes = util.make_caterpillar_graph(n) a0 = nodes[0] a = nodes[-1] grad = tf.gradients([a], [a0])[0] print(tf.get_default_graph().as_graph_def()) order1 = linearize_lib.obtain_linear_order() observed_order1 = [n.name for n in order1] g = tf.get_default_graph() # g.version should track if graph was modified, but it doesn't # https://github.com/tensorflow/tensorflow/issues/14233 num_new_deps = linearize_lib.linearize() assert num_new_deps > 0 order2 = linearize_lib.obtain_linear_order() observed_order2 = [n.name for n in order2] assert observed_order1 == observed_order2 num_new_deps = linearize_lib.linearize() assert num_new_deps == 0
def test_caterpillar_linearize():
tf.reset_default_graph()
n = 5
nodes = util.make_caterpillar_graph(n)
a0 = nodes[0]
a = nodes[-1]
grad = tf.gradients([a], [a0])[0]
print(tf.get_default_graph().as_graph_def())
order1 = linearize_lib.obtain_linear_order()
observed_order1 = [n.name for n in order1]
g = tf.get_default_graph()
# g.version should track if graph was modified, but it doesn't
# https://github.com/tensorflow/tensorflow/issues/14233
num_new_deps = linearize_lib.linearize()
assert num_new_deps > 0
order2 = linearize_lib.obtain_linear_order()
observed_order2 = [n.name for n in order2]
assert observed_order1 == observed_order2
num_new_deps = linearize_lib.linearize()
assert num_new_deps == 0
def test_chain_linearize(): tf.reset_default_graph() n = 5 # create a chain with only a single execution order # using make_chain_tanh_const doesn't work because of "shape_as_tensor" # op that is not constrained # (see "Running ones/shape_as_tensor after ones/Const") nodes = util.make_chain_tanh(n) a0 = nodes[0] a = nodes[-1] order1 = linearize_lib.obtain_linear_order() observed_order1 = [n.name for n in order1] num_new_deps = linearize_lib.linearize(targets=[a]) assert num_new_deps == 0
def convexify_obj(obj):
"""
:param obj: objective of a problem
:return: convexified onjective or None
"""
# not dcp
if obj.is_dcp() == False:
lin = linearize(obj.args[0])
# non-sub/super-diff
if lin is None:
return None
else:
if obj.NAME == 'minimize':
result = Minimize(lin)
else:
result = Maximize(lin)
else:
result = obj
return result
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=y)
loss = tf.reduce_mean(xentropy)
optimizer = tf.train.AdamOptimizer()
training_op = optimizer.minimize(loss)
with tf.device('/cpu:0'):
with tf.name_scope("eval"):
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
with tf.name_scope("init_and_save"):
init = tf.global_variables_initializer()
saver = tf.train.Saver()
linearize_lib.linearize()
graph = tf.get_default_graph()
writer = tf.summary.FileWriter("./simple_graph_events")
writer.add_graph(graph=graph)
n_epochs = 1000
batch_size = 4096
best_loss_val = np.infty
check_interval = 500
checks_since_last_progress = 0
max_checks_without_progress = 20
best_model_params = None
config = tf.ConfigProto()
#go through each group
n = 0
if 0 in group:
groups = [i + 1 for i in range(number_of_groups - 1)] + [0]
else:
groups = [i + 1 for i in range(number_of_groups)]
print(groups)
print(group.keys())
for g in groups:
#sort them by degree
group_edges = []
for edge in edge_list:
if edge[0] in group[g] and edge[1] in group[g]:
group_edges.append(edge)
group_ordered = lin.linearize(group_edges)
for ID in group[g]:
if ID not in group_ordered:
group_ordered.append(ID)
group_ordered.reverse()
# degree=[]
# for ID in group[g]:
# ID_df=df1[(df1['Ant_ID']==ID)|(df1['Ant_ID_(partner)']==ID)]
# deg=len(set(set(pd.concat([ID_df['Ant_ID'],ID_df['Ant_ID_(partner)']]))))
# #loc=list(df[(df['ID']==ID)]['mean_location_'+den])
# #deg=loc[0]
# #degree.append([ID,deg])
# if ID in order:
# degree.append([ID,order[ID]])
# else:
