def testSequenceLoss(self):
with self.test_session() as sess:
output_classes = 5
logits = [tf.constant(i + 0.5, shape=[2, 5]) for i in xrange(3)]
targets = [tf.constant(i, tf.int32, shape=[2]) for i in xrange(3)]
weights = [tf.constant(1.0, shape=[2]) for i in xrange(3)]
average_loss_per_example = tf.nn.seq2seq.sequence_loss(
logits, targets, weights, output_classes,
average_across_timesteps=True,
average_across_batch=True)
res = sess.run(average_loss_per_example)
self.assertAllClose(res, 1.60944)
average_loss_per_sequence = tf.nn.seq2seq.sequence_loss(
logits, targets, weights, output_classes,
average_across_timesteps=False,
average_across_batch=True)
res = sess.run(average_loss_per_sequence)
self.assertAllClose(res, 4.828314)
total_loss = tf.nn.seq2seq.sequence_loss(
logits, targets, weights, output_classes,
average_across_timesteps=False,
average_across_batch=False)
res = sess.run(total_loss)
self.assertAllClose(res, 9.656628)
def unzip(x, split_dim, current_length, num_splits=2, name=None):
"""Splits a tensor by unzipping along the split_dim.
For example the following array split into 2 would be:
[1, 2, 3, 4, 5, 6] -> [1, 3, 5], [2, 4, 6]
and by 3:
[1, 2, 3, 4] -> [1, 4], [2], [3]
Args:
x: The tensor to split.
split_dim: The dimension to split along.
current_length: Current length along the split_dim.
num_splits: The number of splits.
name: Optional name for this op.
Returns:
A length num_splits sequence.
"""
with tf.op_scope([x], name, 'unzip') as scope:
x = tf.convert_to_tensor(x, name='x')
# There is probably a more efficient way to do this.
all_splits = tf.split(split_dim, current_length, x, name=scope)
splits = [[] for _ in xrange(num_splits)]
for i in xrange(current_length):
splits[i % num_splits].append(all_splits[i])
return [tf.concat(split_dim, s) for s in splits]
def test_maintenance_failure_recovery_with_capacity(self):
backend = RedisBackend(capacity=10, truncation_chance=0.0)
t = time.time()
# Add 10 items to the timeline.
for i in xrange(10):
backend.add('timeline', Record('record:{}'.format(i), '{}'.format(i), t + i))
try:
with backend.digest('timeline', 0) as records:
raise Exception('This causes the digest to not be closed.')
except Exception:
pass
# The 10 existing items should now be in the digest set (the exception
# prevented the close operation from occurring, so they were never
# deleted from Redis or removed from the digest set.) If we add 10 more
# items, they should be added to the timeline set (not the digest set.)
for i in xrange(10, 20):
backend.add('timeline', Record('record:{}'.format(i), '{}'.format(i), t + i))
# Maintenance should move the timeline back to the waiting state, ...
backend.maintenance(time.time())
# The schedule should now contain the timeline.
assert set(entry.key for entry in backend.schedule(time.time())) == set(['timeline'])
# Only the new records should exist -- the older one should have been
# trimmed to avoid the digest growing beyond the timeline capacity.
with backend.digest('timeline', 0) as records:
expected_keys = set('record:{}'.format(i) for i in xrange(10, 20))
assert set(record.key for record in records) == expected_keys
def next_batch(self):
enc_batch = np.zeros((self._batch_size, self._enc_timesteps), dtype=np.int32)
enc_input_lens = np.zeros(self._batch_size, dtype=np.int32)
dec_batch = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.int32)
dec_output_lens = np.zeros(self._batch_size, dtype=np.int32)
target_batch = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.int32)
loss_weights = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.float32)
origin_articles = ['None'] * self._batch_size
origin_abstracts = ['None'] * self._batch_size
buckets = self._bucket_input_queue.get()
for i in xrange(self._batch_size):
(enc_inputs, dec_inputs, targets, enc_input_len, dec_output_len, article, abstract) = buckets[i]
origin_articles[i] = article
origin_abstracts[i] = abstract
enc_input_lens[i] = enc_input_len
dec_output_lens[i] = dec_output_len
enc_batch[i, :] = enc_inputs[:]
dec_batch[i, :] = dec_inputs[:]
target_batch[i, :] = targets[:]
for j in xrange(dec_output_len):
loss_weights[i][j] = 1
return (
enc_batch, dec_batch, target_batch, enc_input_lens, dec_output_lens, loss_weights,
origin_articles, origin_abstracts
)
def _shapes(tensor_list_list, shapes, enqueue_many):
"""Calculate and merge the shapes of incoming tensors.
Args:
tensor_list_list: List of tensor lists.
shapes: List of shape tuples corresponding to tensors within the lists.
enqueue_many: Boolean describing whether shapes will be enqueued as
batches or individual entries.
Returns:
A list of shapes aggregating shape inference info from `tensor_list_list`,
or returning `shapes` if it is not `None`.
Raises:
ValueError: If any of the inferred shapes in `tensor_list_list` lack a
well defined rank.
"""
if shapes is None:
len0 = len(tensor_list_list[0])
for tl in tensor_list_list:
for i in xrange(len0):
if tl[i].get_shape().ndims is None:
raise ValueError("Cannot infer Tensor's rank: %s" % tl[i])
shapes = [_merge_shapes(
[tl[i].get_shape().as_list() for tl in tensor_list_list], enqueue_many)
for i in xrange(len0)]
return shapes
def testError(self):
for rank in VALID_FFT_RANKS:
for dims in xrange(0, rank):
x = np.zeros((1,) * dims).astype(np.complex64)
with self.assertRaisesWithPredicateMatch(
ValueError, "Shape must be .*rank {}.*".format(rank)):
self._tfFFT(x, rank)
with self.assertRaisesWithPredicateMatch(
ValueError, "Shape must be .*rank {}.*".format(rank)):
self._tfIFFT(x, rank)
for dims in xrange(rank, rank + 2):
x = np.zeros((1,) * rank)
# Test non-rank-1 fft_length produces an error.
fft_length = np.zeros((1, 1)).astype(np.int32)
with self.assertRaisesWithPredicateMatch(ValueError,
"Shape must be .*rank 1"):
self._tfFFT(x, rank, fft_length)
with self.assertRaisesWithPredicateMatch(ValueError,
"Shape must be .*rank 1"):
self._tfIFFT(x, rank, fft_length)
# Test wrong fft_length length.
fft_length = np.zeros((rank + 1,)).astype(np.int32)
with self.assertRaisesWithPredicateMatch(
ValueError, "Dimension must be .*but is {}.*".format(rank + 1)):
self._tfFFT(x, rank, fft_length)
with self.assertRaisesWithPredicateMatch(
ValueError, "Dimension must be .*but is {}.*".format(rank + 1)):
self._tfIFFT(x, rank, fft_length)
def get_run_op():
# Create an optimizer that performs gradient descent.
#opt = tf.train.GradientDescentOptimizer(learning_rate=0.01)
slice_size = FLAGS.batch_size / FLAGS.num_cuts
print('Slice size:{}'.format(slice_size))
data = None
label = None
last_fc = [tf.no_op()]
with tf.device('/gpu:0'):
data = tf.get_variable(
name = 'data',
shape=[slice_size, FLAGS.hidden_size],
trainable=False)
'''
label = tf.get_variable(
name = 'label',
shape = [slice_size, FLAGS.hidden_size],
trainable=False))
with tf.variable_scope('fc_in'):
weight_in = tf.zeros([1000, FLAGS.hidden_size])
for k in xrange(FLAGS.num_cuts):
with tf.control_dependencies([last_fc[-1]]):
last_fc.append(tf.matmul(data[k+1], weight_in))
'''
for i in xrange(FLAGS.num_cuts):
last_fc.append(data)
for i in xrange(FLAGS.num_layers):
dev = '/gpu:%d' % (i * FLAGS.num_gpus / FLAGS.num_layers)
with tf.device(dev), scopes.arg_scope([variables.variable], device=dev):
tmp_fc = [tf.no_op()]
with tf.variable_scope('fc%d' % i):
w = tf.get_variable(
name='w',
shape=[FLAGS.hidden_size, FLAGS.hidden_size],
trainable=True)
for k in xrange(FLAGS.num_cuts):
with tf.control_dependencies([tmp_fc[-1]]):
tmp_fc.append(tf.matmul(last_fc[k+1], w))
last_fc = tmp_fc
if i == FLAGS.num_layers - 1:
with tf.control_dependencies(last_fc):
train_op = tf.no_op()
'''
with tf.device('/gpu:%d' % (FLAGS.num_gpus - 1)):
tmp_fc = [tf.no_op()]
with tf.variable_scope('fc_out'):
weight_out = tf.zeros([FLAGS.hidden_size, 1000])
for k in xrange(FLAGS.num_cuts):
with tf.control_dependencies([tmp_fc[-1]]):
tmp_fc.append(tf.matmul(last_fc[k+1], weight_out))
last_fc = tmp_fc
loss = tf.nn_softmax_cross_entropy_with_logits(last_fc, labels, name='xentropy')
grads = opt.compute_gradients(loss)
apply_gradient_op = opt.apply_gradients(grads)
train_op = tf.group(apply_gradient_op)
'''
init_op = tf.initialize_all_variables()
return init_op, train_op
def conv2d(self, img, kern, mode="valid"):
"""
Basic slow python implementatation for DebugMode
"""
if not imported_scipy_signal:
raise NotImplementedError(
"AbstractConv perform requires the python package"
" for scipy.signal to be installed.")
if not (mode in ('valid', 'full')):
raise ValueError(
'invalid mode {}, which must be either '
'"valid" or "full"'.format(mode))
out_shape = get_conv_output_shape(img.shape, kern.shape, mode, [1, 1])
out = numpy.zeros(out_shape, dtype=img.dtype)
val = _valfrommode(mode)
bval = _bvalfromboundary('fill')
with warnings.catch_warnings():
warnings.simplefilter('ignore', numpy.ComplexWarning)
for b in xrange(img.shape[0]):
for n in xrange(kern.shape[0]):
for im0 in xrange(img.shape[1]):
# some cast generates a warning here
out[b, n, ...] += _convolve2d(img[b, im0, ...],
kern[n, im0, ...],
1, val, bval, 0)
return out
def eval(self):
"""Evaluate analogy questions and reports accuracy."""
# How many questions we get right at precision@1.
correct = 0
total = self._analogy_questions.shape[0]
start = 0
while start < total:
limit = start + 2500
sub = self._analogy_questions[start:limit, :]
idx = self._predict(sub)
start = limit
for question in xrange(sub.shape[0]):
for j in xrange(4):
if idx[question, j] == sub[question, 3]:
# Bingo! We predicted correctly. E.g., [italy, rome, france, paris].
correct += 1
break
elif idx[question, j] in sub[question, :3]:
# We need to skip words already in the question.
continue
else:
# The correct label is not the precision@1
break
print()
print("Eval %4d/%d accuracy = %4.1f%%" % (correct, total,
correct * 100.0 / total))
def _lost_point_level3(modules, modules_count):
modules_range_short = xrange(modules_count-6)
lost_point = 0
for row in xrange(modules_count):
this_row = modules[row]
for col in modules_range_short:
if (this_row[col]
and not this_row[col + 1]
and this_row[col + 2]
and this_row[col + 3]
and this_row[col + 4]
and not this_row[col + 5]
and this_row[col + 6]):
lost_point += 40
for col in xrange(modules_count):
for row in modules_range_short:
if (modules[row][col]
and not modules[row + 1][col]
and modules[row + 2][col]
and modules[row + 3][col]
and modules[row + 4][col]
and not modules[row + 5][col]
and modules[row + 6][col]):
lost_point += 40
return lost_point
def write_to_buffer(self, buffer, colors=None):
if self.mode == MODE_NUMBER:
for i in xrange(0, len(self.data), 3):
chars = self.data[i:i + 3]
bit_length = NUMBER_LENGTH[len(chars)]
color = self._getColor(i, colors)
buffer.put(int(chars), bit_length, color)
elif self.mode == MODE_ALPHA_NUM:
for i in xrange(0, len(self.data), 2):
chars = self.data[i:i + 2]
color = self._getColor(i, colors)
if len(chars) > 1:
buffer.put(
ALPHA_NUM.find(chars[0]) * 45 +
ALPHA_NUM.find(chars[1]), 11, color)
else:
buffer.put(ALPHA_NUM.find(chars), 6, color)
else:
if six.PY3:
# Iterating a bytestring in Python 3 returns an integer,
# no need to ord().
data = self.data
else:
data = [ord(c) for c in self.data]
for i, c in enumerate(data):
color = self._getColor(i, colors)
buffer.put(c, 8, color)
def next_batch(self, batch_size, fake_data=False):
"""Return the next `batch_size` examples from this data set."""
if fake_data:
fake_image = [1] * 784
if self.one_hot:
fake_label = [1] + [0] * 9
else:
fake_label = 0
return [fake_image for _ in xrange(batch_size)], [
fake_label for _ in xrange(batch_size)]
start = self._index_in_epoch
self._index_in_epoch += batch_size
if self._index_in_epoch > self._num_examples:
# Finished epoch
self._epochs_completed += 1
# Shuffle the data
perm = numpy.arange(self._num_examples)
numpy.random.shuffle(perm)
self._images = self._images[perm]
self._labels = self._labels[perm]
# Start next epoch
start = 0
self._index_in_epoch = batch_size
assert batch_size <= self._num_examples
end = self._index_in_epoch
return self._images[start:end], self._labels[start:end]
def testParallelDequeueUpToRandomPartition(self):
with self.test_session() as sess:
dequeue_sizes = [random.randint(50, 150) for _ in xrange(10)]
total_elements = sum(dequeue_sizes)
q = tf.RandomShuffleQueue(total_elements, 0, tf.float32, shapes=())
elems = [10.0 * x for x in xrange(total_elements)]
enqueue_op = q.enqueue_many((elems,))
dequeue_ops = [q.dequeue_up_to(size) for size in dequeue_sizes]
enqueue_op.run()
# Dequeue random number of items in parallel on 10 threads.
dequeued_elems = []
def dequeue(dequeue_op):
dequeued_elems.extend(sess.run(dequeue_op))
threads = []
for dequeue_op in dequeue_ops:
threads.append(self.checkedThread(target=dequeue, args=(dequeue_op,)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.assertItemsEqual(elems, dequeued_elems)
def visit_node(n, levels):
lbl = "{"
if data is None:
if self.k <= max_k_labeled:
lbl = repr(n.label()).\
replace("{","\{").\
replace("}","\}").\
replace("|","\|").\
replace("<","\<").\
replace(">","\>")
else:
lbl = str(n)
else:
s = self.bucket_to_block(n.bucket)
for i in xrange(self.blocks_per_bucket):
lbl += "{%s}" % (data[s+i])
if i + 1 != self.blocks_per_bucket:
lbl += "|"
lbl += "}"
f.write(" %s [penwidth=%s,label=\"%s\"];\n"
% (n.bucket, 1, lbl))
levels += 1
if (max_levels is None) or (levels <= max_levels):
for i in xrange(self.k):
cn = n.child_node(i)
if not self.is_nil_node(cn):
visit_node(cn, levels)
f.write(" %s -> %s ;\n" % (n.bucket, cn.bucket))
def show_topics(self, topics=10, topn=10, log=False, formatted=True):
shown = []
if topics < 0:
topics = len(self.data)
topics = min(topics, len(self.data))
for k in xrange(topics):
lambdak = list(self.data[k, :])
lambdak = lambdak / sum(lambdak)
temp = zip(lambdak, xrange(len(lambdak)))
temp = sorted(temp, key=lambda x: x[0], reverse=True)
topic_terms = self.show_topic_terms(temp, topn)
if formatted:
topic = self.format_topic(k, topic_terms)
# assuming we only output formatted topics
if log:
logger.info(topic)
else:
topic = (k, topic_terms)
shown.append(topic)
return shown
def testConv3DTransposeSame(self):
with self.test_session():
strides = [1, 2, 2, 2, 1]
# Input, output: [batch, depth, height, width, depth]
x_shape = [2, 5, 6, 4, 3]
y_shape = [2, 10, 12, 8, 2]
# Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth]
f_shape = [3, 3, 3, 2, 3]
x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides,
padding="SAME")
value = output.eval()
for n in xrange(x_shape[0]):
for k in xrange(f_shape[3]):
for w in xrange(y_shape[3]):
for h in xrange(y_shape[2]):
for d in xrange(y_shape[1]):
# We add a case for locations divisible by the stride.
d_in = d % strides[1] == 0 and 0 < d < y_shape[1] - 1
h_in = h % strides[2] == 0 and 0 < h < y_shape[2] - 1
w_in = w % strides[3] == 0 and 0 < w < y_shape[3] - 1
if d_in + h_in + w_in == 3:
target = 8 * 3.0
elif d_in + h_in + w_in == 2:
target = 4 * 3.0
elif d_in or h_in or w_in:
target = 2 * 3.0
else:
target = 3.0
self.assertAllClose(target, value[n, d, h, w, k])
def add_lines(self, levels, colors, linewidths, erase=True):
'''
Draw lines on the colorbar.
*colors* and *linewidths* must be scalars or
sequences the same length as *levels*.
Set *erase* to False to add lines without first
removing any previously added lines.
'''
y = self._locate(levels)
igood = (y < 1.001) & (y > -0.001)
y = y[igood]
if cbook.iterable(colors):
colors = np.asarray(colors)[igood]
if cbook.iterable(linewidths):
linewidths = np.asarray(linewidths)[igood]
N = len(y)
x = np.array([0.0, 1.0])
X, Y = np.meshgrid(x, y)
if self.orientation == 'vertical':
xy = [list(zip(X[i], Y[i])) for i in xrange(N)]
else:
xy = [list(zip(Y[i], X[i])) for i in xrange(N)]
col = collections.LineCollection(xy, linewidths=linewidths)
if erase and self.lines:
for lc in self.lines:
lc.remove()
self.lines = []
self.lines.append(col)
col.set_color(colors)
self.ax.add_collection(col)
self.stale = True
def testConv2DTransposeSingleStride(self):
with self.test_session():
strides = [1, 1, 1, 1]
# Input, output: [batch, height, width, depth]
x_shape = [2, 6, 4, 3]
y_shape = [2, 6, 4, 2]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
output = tf.nn.conv2d_transpose(x, f, y_shape, strides=strides,
padding="SAME")
value = output.eval()
# We count the number of cells being added at the locations in the output.
# At the center, #cells=kernel_height * kernel_width
# At the corners, #cells=ceil(kernel_height/2) * ceil(kernel_width/2)
# At the borders, #cells=ceil(kernel_height/2)*kernel_width or
# kernel_height * ceil(kernel_width/2)
for n in xrange(x_shape[0]):
for k in xrange(f_shape[2]):
for w in xrange(y_shape[2]):
for h in xrange(y_shape[1]):
target = 4 * 3.0
h_in = h > 0 and h < y_shape[1] - 1
w_in = w > 0 and w < y_shape[2] - 1
if h_in and w_in:
target += 5 * 3.0
elif h_in or w_in:
target += 2 * 3.0
self.assertAllClose(target, value[n, h, w, k])
def testConv2DTransposeSame(self):
with self.test_session():
strides = [1, 2, 2, 1]
# Input, output: [batch, height, width, depth]
x_shape = [2, 6, 4, 3]
y_shape = [2, 12, 8, 2]
# Filter: [kernel_height, kernel_width, output_depth, input_depth]
f_shape = [3, 3, 2, 3]
x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
output = tf.nn.conv2d_transpose(x, f, y_shape, strides=strides,
padding="SAME")
value = output.eval()
for n in xrange(x_shape[0]):
for k in xrange(f_shape[2]):
for w in xrange(y_shape[2]):
for h in xrange(y_shape[1]):
target = 3.0
# We add a case for locations divisible by the stride.
h_in = h % strides[1] == 0 and h > 0 and h < y_shape[1] - 1
w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
if h_in and w_in:
target += 9.0
elif h_in or w_in:
target += 3.0
self.assertAllClose(target, value[n, h, w, k])
def populate(parent, howmany, max_children):
if howmany > max_children:
children = randint(2, max_children)
distribution = []
for _ in xrange(0, children - 1):
distribution.append(int(howmany / children))
distribution.append(howmany - sum(distribution, 0))
for i in xrange(0, children):
steal_target = randint(0, children - 1)
while steal_target == i:
steal_target = randint(0, children -1)
steal_count = int(randint(-1 * distribution[i],
distribution[steal_target]) / 2)
distribution[i] += steal_count
distribution[steal_target] -= steal_count
for i in xrange(0, children):
make_dict = randint(0, 1)
if make_dict:
baby = {}
else:
baby = []
populate(baby, distribution[i], max_children)
if isinstance(parent, dict):
key = os.urandom(8)
key = "".join(chr(c) for c in hexlify(key)) \
if PY3 else key.encode("hex")
parent[key] = baby
else:
parent.append(baby)
else:
populate_with_leaves(parent, howmany)
def test_simple():
fig = plt.figure()
# un-comment to debug
# recursive_pickle(fig)
pickle.dump(fig, BytesIO(), pickle.HIGHEST_PROTOCOL)
ax = plt.subplot(121)
pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)
ax = plt.axes(projection='polar')
plt.plot(list(xrange(10)), label='foobar')
plt.legend()
# recursive_pickle(fig)
pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)
# ax = plt.subplot(121, projection='hammer')
# recursive_pickle(ax, 'figure')
# pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)
plt.figure()
plt.bar(left=list(xrange(10)), height=list(xrange(10)))
pickle.dump(plt.gca(), BytesIO(), pickle.HIGHEST_PROTOCOL)
fig = plt.figure()
ax = plt.axes()
plt.plot(list(xrange(10)))
ax.set_yscale('log')
pickle.dump(fig, BytesIO(), pickle.HIGHEST_PROTOCOL)
def parse_atoms(self, tokens, command, min_size, max_size=None):
"""
Parses a sequence of N atoms (min_size <= N <= max_size) consuming
the tokens
"""
if max_size is None:
max_size = min_size
res = []
current = None
for _ in xrange(min_size):
current = next(tokens)
if current == ")":
raise SyntaxError("Expected at least %d arguments in %s command." % (min_size, command))
if current == "(":
raise SyntaxError("Unexpected token '(' in %s command." % command)
res.append(current)
for _ in xrange(min_size, max_size + 1):
current = next(tokens)
if current == ")":
return res
if current == "(":
raise SyntaxError("Unexpected token '(' in %s command." % command)
res.append(current)
raise SyntaxError(
"Unexpected token '%s' in %s command. Expected at " "most %d arguments." % (current, command, max_size)
)
def xfun(n, d=None):
""" Create a QTT-representation of 0:prod(n) _vector
call examples:
tt.xfun(2, 5) # create 2 x 2 x 2 x 2 x 2 TT-vector
tt.xfun(3) # create [0, 1, 2] one-dimensional TT-vector
tt.xfun([3, 5, 7], 2) # create 3 x 5 x 7 x 3 x 5 x 7 TT-vector
"""
if isinstance(n, six.integer_types):
n = [n]
if d is None:
n0 = _np.asanyarray(n, dtype=_np.int32)
else:
n0 = _np.array(n * d, dtype=_np.int32)
d = n0.size
if d == 1:
return _vector.vector.from_list(
[_np.reshape(_np.arange(n0[0]), (1, n0[0], 1))])
cr = []
cur_core = _np.ones((1, n0[0], 2))
cur_core[0, :, 0] = _np.arange(n0[0])
cr.append(cur_core)
ni = float(n0[0])
for i in xrange(1, d - 1):
cur_core = _np.zeros((2, n0[i], 2))
for j in xrange(n0[i]):
cur_core[:, j, :] = _np.eye(2)
cur_core[1, :, 0] = ni * _np.arange(n0[i])
ni *= n0[i]
cr.append(cur_core)
cur_core = _np.ones((2, n0[d - 1], 1))
cur_core[1, :, 0] = ni * _np.arange(n0[d - 1])
cr.append(cur_core)
return _vector.vector.from_list(cr)
def test_neville2d(self):
funcx = numpy.sin
funcy = numpy.exp
nrow = 10
ncol = 10
tol = 1.0e-4
# TODO: As with test_neville; can this not be simplified with
# vectorized code
x = numpy.zeros((nrow, ))
y = numpy.zeros((ncol, ))
fval = numpy.empty((nrow, ncol))
row_tmp = numpy.pi / nrow
# col_tmp = 1.0 / float(ncol)
for row in xrange(nrow):
x[row] = (row + 1.0) * row_tmp
for col in xrange(ncol):
y[col] = (col + 1.0) / float(ncol)
fval[row][col] = funcx(x[row]) * funcy(y[col])
for row in xrange(ncol):
xx = (-0.1 + (row + 1.0) / float(nrow)) * numpy.pi
for col in xrange(4):
yy = -0.1 + (col + 1.0) / float(ncol)
answer = funcx(xx) * funcy(yy)
val = utils.neville2d(xx, yy, x, y, fval)
self.assertTrue(utils.Knuth_close(answer, val, tol))
def filter(self, im):
falloff = self.falloff
extent = self.extent
def length(start, end):
start_x, start_y = start
end_x, end_y = end
dist_x = end_x - start_x
dist_y = end_y - start_y
return math.sqrt((dist_x ** 2) + (dist_y ** 2))
def light_falloff(radius, outside):
return ((radius / outside) ** falloff) * extent
im = im.convert('RGBA')
w, h = im.size
center = w / 2, h / 2
outside = length(center, (0, 0))
data = []
for y in xrange(h):
for x in xrange(w):
radius = length(center, (x, y))
factor = light_falloff(radius, outside)
data.append(factor)
alpha_im = Image.new('L', im.size)
alpha_im.putdata(data)
overlay_im = Image.new('L', im.size, 'black')
return Image.composite(overlay_im, im, alpha_im)
def testParams(self):
"""Tests that the params work as intended."""
num_classes = 2
with self.test_session() as sess:
# Experiment 1. Update weights only.
data = constant_op.constant(self.data, dtype=dtypes.float32)
gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
[[3.0, 3.0], [0.0, 0.0]], 'w')
training_ops = gmm_tool.training_ops()
variables.global_variables_initializer().run()
sess.run(gmm_tool.init_ops())
for _ in xrange(self.iterations):
sess.run(training_ops)
# Only the probability to each class is updated.
alphas = sess.run(gmm_tool.alphas())
self.assertGreater(alphas[1], 0.6)
means = sess.run(gmm_tool.clusters())
np.testing.assert_almost_equal(
np.expand_dims([[3.0, 3.0], [0.0, 0.0]], 1), means)
covs = sess.run(gmm_tool.covariances())
np.testing.assert_almost_equal(covs[0], covs[1])
# Experiment 2. Update means and covariances.
gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
[[3.0, 3.0], [0.0, 0.0]], 'mc')
training_ops = gmm_tool.training_ops()
variables.global_variables_initializer().run()
sess.run(gmm_tool.init_ops())
for _ in xrange(self.iterations):
sess.run(training_ops)
alphas = sess.run(gmm_tool.alphas())
self.assertAlmostEqual(alphas[0], alphas[1])
means = sess.run(gmm_tool.clusters())
np.testing.assert_almost_equal(
np.expand_dims([[2.0, 2.0], [-1.0, -1.0]], 1), means, decimal=1)
covs = sess.run(gmm_tool.covariances())
np.testing.assert_almost_equal(
[[0.371111, -0.0050774], [-0.0050774, 0.8651744]], covs[0], decimal=4)
np.testing.assert_almost_equal(
[[0.146976, 0.0259463], [0.0259463, 0.2543971]], covs[1], decimal=4)
# Experiment 3. Update covariances only.
gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
[[-1.0, -1.0], [1.0, 1.0]], 'c')
training_ops = gmm_tool.training_ops()
variables.global_variables_initializer().run()
sess.run(gmm_tool.init_ops())
for _ in xrange(self.iterations):
sess.run(training_ops)
alphas = sess.run(gmm_tool.alphas())
self.assertAlmostEqual(alphas[0], alphas[1])
means = sess.run(gmm_tool.clusters())
np.testing.assert_almost_equal(
np.expand_dims([[-1.0, -1.0], [1.0, 1.0]], 1), means)
covs = sess.run(gmm_tool.covariances())
np.testing.assert_almost_equal(
[[0.1299582, 0.0435872], [0.0435872, 0.2558578]], covs[0], decimal=5)
np.testing.assert_almost_equal(
[[3.195385, 2.6989155], [2.6989155, 3.3881593]], covs[1], decimal=5)
def SoftmaxEval(self, sess, model, num_steps):
"""Evaluate a model in softmax mode.
Adds char, word recall and sequence error rate events to the sw summary
writer, and returns them as well
TODO(rays) Add LogisticEval.
Args:
sess: A tensor flow Session.
model: The model to run in the session. Requires a VGSLImageModel or any
other class that has a using_ctc attribute and a RunAStep(sess) method
that reurns a softmax result with corresponding labels.
num_steps: Number of steps to evaluate for.
Returns:
ErrorRates named tuple.
Raises:
ValueError: If an unsupported number of dimensions is used.
"""
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Run the requested number of evaluation steps, gathering the outputs of the
# softmax and the true labels of the evaluation examples.
total_label_counts = ec.ErrorCounts(0, 0, 0, 0)
total_word_counts = ec.ErrorCounts(0, 0, 0, 0)
sequence_errors = 0
for _ in xrange(num_steps):
softmax_result, labels = model.RunAStep(sess)
# Collapse softmax to same shape as labels.
predictions = softmax_result.argmax(axis=-1)
# Exclude batch from num_dims.
num_dims = len(predictions.shape) - 1
batch_size = predictions.shape[0]
null_label = softmax_result.shape[-1] - 1
for b in xrange(batch_size):
if num_dims == 2:
# TODO(rays) Support 2-d data.
raise ValueError('2-d label data not supported yet!')
else:
if num_dims == 1:
pred_batch = predictions[b, :]
labels_batch = labels[b, :]
else:
pred_batch = [predictions[b]]
labels_batch = [labels[b]]
text = self.StringFromCTC(pred_batch, model.using_ctc, null_label)
truth = self.StringFromCTC(labels_batch, False, null_label)
# Note that recall_errs is false negatives (fn) aka drops/deletions.
# Actual recall would be 1-fn/truth_words.
# Likewise precision_errs is false positives (fp) aka adds/insertions.
# Actual precision would be 1-fp/ocr_words.
total_word_counts = ec.AddErrors(total_word_counts,
ec.CountWordErrors(text, truth))
total_label_counts = ec.AddErrors(total_label_counts,
ec.CountErrors(text, truth))
if text != truth:
sequence_errors += 1
coord.request_stop()
coord.join(threads)
return ec.ComputeErrorRates(total_label_counts, total_word_counts,
sequence_errors, num_steps * batch_size)
def delta(n, d=None, center=0):
""" Create TT-vector for delta-function :math:`\\delta(x - x_0)`. """
if isinstance(n, six.integer_types):
n = [n]
if d is None:
n0 = _np.asanyarray(n, dtype=_np.int32)
else:
n0 = _np.array(n * d, dtype=_np.int32)
d = n0.size
if center < 0:
cind = [0] * d
else:
cind = []
for i in xrange(d):
cind.append(center % n0[i])
center //= n0[i]
if center > 0:
cind = [0] * d
cr = []
for i in xrange(d):
cur_core = _np.zeros((1, n0[i], 1))
cur_core[0, cind[i], 0] = 1
cr.append(cur_core)
return _vector.vector.from_list(cr)
def testTFSummaryScalar(self):
"""Verify processing of tf.summary.scalar."""
event_sink = _EventGenerator(self, zero_out_timestamps=True)
writer = SummaryToEventTransformer(event_sink)
with self.test_session() as sess:
ipt = array_ops.placeholder(dtypes.float32)
summary_lib.scalar('scalar1', ipt)
summary_lib.scalar('scalar2', ipt * ipt)
merged = summary_lib.merge_all()
writer.add_graph(sess.graph)
for i in xrange(10):
summ = sess.run(merged, feed_dict={ipt: i})
writer.add_summary(summ, global_step=i)
accumulator = ea.EventAccumulator(event_sink)
accumulator.Reload()
seq1 = [ea.ScalarEvent(wall_time=0, step=i, value=i) for i in xrange(10)]
seq2 = [
ea.ScalarEvent(
wall_time=0, step=i, value=i * i) for i in xrange(10)
]
self.assertTagsEqual(accumulator.Tags(), {
ea.SCALARS: ['scalar1', 'scalar2'],
ea.GRAPH: True,
ea.META_GRAPH: False,
})
self.assertEqual(accumulator.Scalars('scalar1'), seq1)
self.assertEqual(accumulator.Scalars('scalar2'), seq2)
first_value = accumulator.Scalars('scalar1')[0].value
self.assertTrue(isinstance(first_value, float))
def back(self, expr, model=None):
"""Convert a Z3 expression back into a pySMT expression.
This is done using the Z3 API. For very big expressions, it is
sometimes faster to go through the SMT-LIB format. In those
cases, consider using the method back_via_smtlib.
"""
stack = [expr]
while len(stack) > 0:
current = stack.pop()
key = (askey(current), model)
if key not in self._back_memoization:
self._back_memoization[key] = None
stack.append(current)
for i in xrange(current.num_args()):
stack.append(current.arg(i))
elif self._back_memoization[key] is None:
args = [self._back_memoization[(askey(current.arg(i)), model)]
for i in xrange(current.num_args())]
res = self._back_single_term(current, args, model)
self._back_memoization[key] = res
else:
# we already visited the node, nothing else to do
pass
return self._back_memoization[(askey(expr), model)]
def _GenerateUnorderedInputs(self, size, n):
inputs = [tf.random_uniform(shape=[size]) for _ in xrange(n)]
random.shuffle(inputs)
return inputs
def get_data(self, how_many, offset, model_settings, background_frequency,
background_volume_range, time_shift, mode, sess):
"""Gather samples from the data set, applying transformations as needed.
When the mode is 'training', a random selection of samples will be returned,
otherwise the first N clips in the partition will be used. This ensures that
validation always uses the same samples, reducing noise in the metrics.
Args:
how_many: Desired number of samples to return. -1 means the entire
contents of this partition.
offset: Where to start when fetching deterministically.
model_settings: Information about the current model being trained.
background_frequency: How many clips will have background noise, 0.0 to
1.0.
background_volume_range: How loud the background noise will be.
time_shift: How much to randomly shift the clips by in time.
mode: Which partition to use, must be 'training', 'validation', or
'testing'.
sess: TensorFlow session that was active when processor was created.
Returns:
List of sample data for the transformed samples, and list of labels in
one-hot form.
"""
# Pick one of the partitions to choose samples from.
candidates = self.data_index[mode]
if how_many == -1:
sample_count = len(candidates)
else:
sample_count = max(0, min(how_many, len(candidates) - offset))
# Data and labels will be populated and returned.
if model_settings['raw_data']:
data = np.zeros((sample_count, model_settings['sample_rate']))
else:
data = np.zeros((sample_count, model_settings['fingerprint_size']))
labels = np.zeros((sample_count, model_settings['label_count']))
names = []
desired_samples = model_settings['desired_samples']
use_background = self.background_data and (mode == 'training')
pick_deterministically = (mode != 'training')
# Use the processing graph we created earlier to repeatedly to generate the
# final output sample data we'll use in training.
for i in xrange(offset, offset + sample_count):
# Pick which audio sample to use.
if how_many == -1 or pick_deterministically:
sample_index = i
else:
sample_index = np.random.randint(len(candidates))
sample = candidates[sample_index]
# If we're time shifting, set up the offset for this sample.
if time_shift > 0:
time_shift_amount = np.random.randint(-time_shift, time_shift)
else:
time_shift_amount = 0
if time_shift_amount > 0:
time_shift_padding = [[time_shift_amount, 0], [0, 0]]
time_shift_offset = [0, 0]
else:
time_shift_padding = [[0, -time_shift_amount], [0, 0]]
time_shift_offset = [-time_shift_amount, 0]
input_dict = {
self.wav_filename_placeholder_: sample['file'],
self.time_shift_padding_placeholder_: time_shift_padding,
self.time_shift_offset_placeholder_: time_shift_offset,
}
# Choose a section of background noise to mix in.
if use_background:
background_index = np.random.randint(len(self.background_data))
background_samples = self.background_data[background_index]
background_offset = np.random.randint(
0, len(background_samples) - model_settings['desired_samples'])
background_clipped = background_samples[background_offset:(
background_offset + desired_samples)]
background_reshaped = background_clipped.reshape([desired_samples, 1])
if np.random.uniform(0, 1) < background_frequency:
background_volume = np.random.uniform(0, background_volume_range)
else:
background_volume = 0
else:
background_reshaped = np.zeros([desired_samples, 1])
background_volume = 0
input_dict[self.background_data_placeholder_] = background_reshaped
input_dict[self.background_volume_placeholder_] = background_volume
# If we want silence, mute out the main sample but leave the background.
if sample['label'] == SILENCE_LABEL:
input_dict[self.foreground_volume_placeholder_] = 0
else:
input_dict[self.foreground_volume_placeholder_] = 1
# Run the graph to produce the output audio.
data[i - offset, :] = sess.run(self.mfcc_, feed_dict=input_dict).flatten()
names.append(sample['label'])
label_index = self.word_to_index[sample['label']]
labels[i - offset, label_index] = 1
return data, labels, names
def model_fn_body_sharded(self, sharded_features):
train = self._hparams.mode == tf.contrib.learn.ModeKeys.TRAIN
dp = self._data_parallelism
hparams = self._hparams
def flatten(inputs):
return tf.expand_dims(common_layers.flatten4d3d(inputs), axis=2)
inputs = dp(flatten, sharded_features["inputs"])
inputs_pad = dp(slicenet.embedding_to_padding, inputs)
inputs_mask = dp(lambda x: 1.0 - x, inputs_pad)
inputs_encoded = dp(common_layers.add_timing_signal, inputs)
expert_loss = 0.0
for i in xrange(hparams.num_hidden_layers):
with tf.variable_scope("enc_layer_%d" % i):
inputs_encoded, moe_loss = conv_experts(
inputs_encoded, hparams, dp, self._ps_devices, "SAME",
inputs_mask, i)
expert_loss += tf.reduce_mean(moe_loss) * hparams.moe_loss_coef
# If we're just predicing a class, there is no use for a decoder, return.
if isinstance(hparams.problems[self._problem_idx].target_modality,
modalities.ClassLabelModality):
return inputs_encoded, tf.reduce_mean(expert_loss)
# Decoder.
inputs3d = dp(tf.squeeze, inputs, 2)
inputs_encoded3d = dp(tf.squeeze, inputs_encoded, 2)
encoder_padding = dp(common_attention.embedding_to_padding, inputs3d)
encoder_attention_bias = dp(
common_attention.attention_bias_ignore_padding, encoder_padding)
targets = dp(common_layers.flatten4d3d, sharded_features["targets"])
target_space_emb = dp(slicenet.embed_target_space,
sharded_features["target_space_id"],
hparams.hidden_size)
(decoder_input,
decoder_self_attention_bias) = dp(prepare_decoder, targets,
target_space_emb)
x = dp(tf.nn.dropout, decoder_input, 1.0 - hparams.dropout)
for layer in xrange(hparams.num_hidden_layers):
with tf.variable_scope("dec_layer_%d" % layer):
with tf.variable_scope("attention"):
y = dp(common_attention.multihead_attention,
x,
None,
decoder_self_attention_bias,
hparams.hidden_size,
hparams.hidden_size,
hparams.hidden_size,
hparams.num_heads,
hparams.attention_dropout,
name="decoder_self_attention")
z = dp(common_attention.multihead_attention,
y,
inputs_encoded3d,
encoder_attention_bias,
hparams.hidden_size,
hparams.hidden_size,
hparams.hidden_size,
hparams.num_heads,
hparams.attention_dropout,
name="encdec_attention")
x = dp(residual_fn3, x, y, z, hparams)
with tf.variable_scope("ffn"):
if str(layer) in hparams.moe_layers.split(","):
y, moe_loss = common_layers.moe_layer(
dp, self._ps_devices, x, train,
hparams.hidden_size, hparams.filter_size,
hparams.moe_n1, hparams.moe_n2,
hparams.moe_loss_coef)
expert_loss += tf.reduce_mean(moe_loss)
else:
y = dp(common_layers.conv_hidden_relu,
x,
hparams.filter_size,
hparams.hidden_size,
dropout=hparams.dropout)
x = dp(residual_fn2, x, y, hparams)
x = dp(tf.expand_dims, x, 2)
return x, tf.reduce_mean(expert_loss)
def embedding_lookup(params,
ids,
partition_strategy="mod",
name=None,
validate_indices=True):
"""Looks up `ids` in a list of embedding tensors.
This function is used to perform parallel lookups on the list of
tensors in `params`. It is a generalization of
[`tf.gather()`](../../api_docs/python/array_ops.md#gather), where `params` is
interpreted as a partitioning of a large embedding tensor. `params` may be
a `PartitionedVariable` as returned by using `tf.get_variable()` with a
partitioner.
If `len(params) > 1`, each element `id` of `ids` is partitioned between
the elements of `params` according to the `partition_strategy`.
In all strategies, if the id space does not evenly divide the number of
partitions, each of the first `(max_id + 1) % len(params)` partitions will
be assigned one more id.
If `partition_strategy` is `"mod"`, we assign each id to partition
`p = id % len(params)`. For instance,
13 ids are split across 5 partitions as:
`[[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8], [4, 9]]`
If `partition_strategy` is `"div"`, we assign ids to partitions in a
contiguous manner. In this case, 13 ids are split across 5 partitions as:
`[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10], [11, 12]]`
The results of the lookup are concatenated into a dense
tensor. The returned tensor has shape `shape(ids) + shape(params)[1:]`.
Args:
params: A list of tensors with the same type and which can be concatenated
along dimension 0. Alternatively, a `PartitionedVariable`, created by
partitioning along dimension 0. Each element must be appropriately sized
for the given `partition_strategy`.
ids: A `Tensor` with type `int32` or `int64` containing the ids to be looked
up in `params`.
partition_strategy: A string specifying the partitioning strategy, relevant
if `len(params) > 1`. Currently `"div"` and `"mod"` are supported. Default
is `"mod"`.
name: A name for the operation (optional).
validate_indices: Whether or not to validate gather indices.
Returns:
A `Tensor` with the same type as the tensors in `params`.
Raises:
ValueError: If `params` is empty.
"""
if params is None or params == []: # pylint: disable=g-explicit-bool-comparison
raise ValueError("Need at least one param")
if isinstance(params, variables.PartitionedVariable):
params = list(params) # Iterate to get the underlying Variables.
if not isinstance(params, list):
params = [params]
with ops.name_scope(name, "embedding_lookup", params + [ids]) as name:
np = len(params) # Number of partitions
params = ops.convert_n_to_tensor_or_indexed_slices(params,
name="params")
if np == 1:
with ops.colocate_with(params[0]):
return array_ops.gather(params[0],
ids,
name=name,
validate_indices=validate_indices)
else:
ids = ops.convert_to_tensor(ids, name="ids")
flat_ids = array_ops.reshape(ids, [-1])
original_indices = math_ops.range(array_ops.size(flat_ids))
# Create p_assignments and set new_ids depending on the strategy.
if partition_strategy == "mod":
p_assignments = flat_ids % np
new_ids = flat_ids // np
elif partition_strategy == "div":
# Compute num_total_ids as the sum of dim-0 of params, then assign to
# partitions based on a constant number of ids per partition. Optimize
# if we already know the full shape statically.
dim_0_size = params[0].get_shape()[0]
for p in xrange(1, np):
dim_0_size += params[p].get_shape()[0]
if dim_0_size.value:
num_total_ids = constant_op.constant(
dim_0_size.value, flat_ids.dtype)
else:
dim_0_sizes = []
for p in xrange(np):
if params[p].get_shape()[0].value is not None:
dim_0_sizes.append(params[p].get_shape()[0].value)
else:
with ops.colocate_with(params[p]):
dim_0_sizes.append(
array_ops.shape(params[p])[0])
num_total_ids = math_ops.reduce_sum(
math_ops.cast(array_ops.pack(dim_0_sizes),
flat_ids.dtype))
ids_per_partition = num_total_ids // np
extras = num_total_ids % np
p_assignments = math_ops.maximum(
flat_ids // (ids_per_partition + 1),
(flat_ids - extras) // ids_per_partition)
# Emulate a conditional using a boolean indicator tensor
is_in_first_extras_partitions = math_ops.cast(
p_assignments < extras, flat_ids.dtype)
new_ids = (is_in_first_extras_partitions *
(flat_ids % (ids_per_partition + 1)) +
(1 - is_in_first_extras_partitions) *
((flat_ids - extras) % ids_per_partition))
else:
raise ValueError("Unrecognized partition strategy: " +
partition_strategy)
# Cast partition assignments to int32 for use in dynamic_partition.
# There really should not be more than 2^32 partitions.
p_assignments = math_ops.cast(p_assignments, dtypes.int32)
# Partition list of ids based on assignments into np separate lists
gather_ids = data_flow_ops.dynamic_partition(
new_ids, p_assignments, np)
# Similarly, partition the original indices.
pindices = data_flow_ops.dynamic_partition(original_indices,
p_assignments, np)
# Do np separate lookups, finding embeddings for plist[p] in params[p]
partitioned_result = []
for p in xrange(np):
with ops.colocate_with(params[p]):
partitioned_result.append(
array_ops.gather(params[p],
gather_ids[p],
validate_indices=validate_indices))
# Stitch these back together
ret = data_flow_ops.dynamic_stitch(pindices,
partitioned_result,
name=name)
# Reshape to reverse the flattening of ids.
element_shape = params[0].get_shape()[1:]
for p in params[1:]:
element_shape = element_shape.merge_with(p.get_shape()[1:])
if element_shape.is_fully_defined():
ret = array_ops.reshape(
ret,
array_ops.concat(0, [array_ops.shape(ids), element_shape]))
else:
# It's important that we compute params[0].shape on the right device
# to avoid data motion.
with ops.colocate_with(params[0]):
params_shape = array_ops.shape(params[0])
ret = array_ops.reshape(
ret,
array_ops.concat(0, [
array_ops.shape(ids),
array_ops.slice(params_shape, [1], [-1])
]))
# output shape = ids.shape + params[*].shape[1:]
# Normally the reshape is sufficient, but setting shape explicitly
# teaches shape inference that params[1:].get_shape() matters.
ret.set_shape(ids.get_shape().concatenate(element_shape))
return ret
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 7, 7, 3], name="z")
pred_annotation, logits = inference(image, keep_probability, z)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
tf.summary.image("pred_annotation",
tf.cast(pred_annotation, tf.uint8),
max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
loss = tf.reduce_mean(tf.squared_difference(logits, annotation))
loss_summary = tf.summary.scalar("entropy", loss)
grads = train_z(loss, z)
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
xx = random.randint(0, 100)
xy = random.randint(0, 100)
h = random.randint(50, 100)
w = random.randint(50, 100)
train_images[:, xx:xx + h, xy:xy + w, :] = 0
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 7, 7, 3))
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_
}
#train_images[:,50:100,50:100,:] =0
v = 0
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
z_loss, summ = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
xx = random.randint(50, 100)
xy = random.randint(50, 100)
h = random.randint(50, 100)
w = random.randint(50, 100)
valid_images[:, xx:xx + h, xy:xy + w, :] = 0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
saver.save(sess, FLAGS.logs_dir + "model_z.ckpt", 500)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
2)
xx = random.randint(50, 100)
xy = random.randint(50, 100)
h = random.randint(50, 100)
w = random.randint(50, 100)
valid_images[:, xx:xx + h, xy:xy + w, :] = 0
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 7, 7, 3))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 0.85,
z: z_
}
v = 0
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict)
print("z_step: %d, Train_loss:%g" % (p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -1.0, 1.0)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
keep_probability: 1.0
})
# valid_annotations = np.squeeze(valid_annotations, axis=3)
# pred = np.squeeze(pred, axis=3)
print(valid_images.shape)
print(valid_annotations.shape)
print(pred.shape)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(valid_annotations[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="pred_" + str(5 + itr))
print("Saved image: %d" % itr)
def run_training():
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
# Create model directory
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
rgb_pre_model_save_dir = "/home/project/I3D/I3D/checkpoints/rgb_scratch"
with tf.Graph().as_default():
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
FLAGS.crop_size, FLAGS.rgb_channels, FLAGS.flow_channels)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
global_step,
decay_steps=3000,
decay_rate=0.1,
staircase=True)
opt_rgb = tf.train.AdamOptimizer(learning_rate)
#opt_stable = tf.train.MomentumOptimizer(learning_rate, 0.9)
with tf.variable_scope('RGB'):
rgb_logit, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits')(rgb_images_placeholder, is_training)
rgb_loss = tower_loss(rgb_logit, labels_placeholder)
accuracy = tower_acc(rgb_logit, labels_placeholder)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
rgb_grads = opt_rgb.compute_gradients(rgb_loss)
apply_gradient_rgb = opt_rgb.apply_gradients(
rgb_grads, global_step=global_step)
train_op = tf.group(apply_gradient_rgb)
null_op = tf.no_op()
# Create a saver for loading trained checkpoints.
rgb_variable_map = {}
for variable in tf.global_variables():
if variable.name.split(
'/')[0] == 'RGB' and 'Adam' not in variable.name.split(
'/')[-1] and variable.name.split('/')[2] != 'Logits':
#rgb_variable_map[variable.name.replace(':0', '')[len('RGB/inception_i3d/'):]] = variable
rgb_variable_map[variable.name.replace(':0', '')] = variable
rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
# Create summary writter
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('rgb_loss', rgb_loss)
tf.summary.scalar('learning_rate', learning_rate)
merged = tf.summary.merge_all()
# load pre_train models
ckpt = tf.train.get_checkpoint_state(rgb_pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
rgb_saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
train_writer = tf.summary.FileWriter(
'./visual_logs/train_rgb_scratch_10000_6_64_0.0001_decay', sess.graph)
test_writer = tf.summary.FileWriter(
'./visual_logs/test_rgb_scratch_10000_6_64_0.0001_decay', sess.graph)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
rgb_train_images, flow_train_images, train_labels, _, _, _ = input_data.read_clip_and_label(
filename='../../list/ucf_list/train_flow.list',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
sess.run(train_op,
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: True
})
duration = time.time() - start_time
print('Step %d: %.3f sec' % (step, duration))
# Save a checkpoint and evaluate the model periodically.
if step % 10 == 0 or (step + 1) == FLAGS.max_steps:
print('Training Data Eval:')
summary, acc, loss_rgb = sess.run(
[merged, accuracy, rgb_loss],
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
print("rgb_loss: " + "{:.5f}".format(loss_rgb))
train_writer.add_summary(summary, step)
print('Validation Data Eval:')
rgb_val_images, flow_val_images, val_labels, _, _, _ = input_data.read_clip_and_label(
filename='../../list/ucf_list/test_flow.list',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
rgb_images_placeholder: rgb_val_images,
labels_placeholder: val_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
test_writer.add_summary(summary, step)
if (step + 1) % 3000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess,
os.path.join(model_save_dir, 'i3d_ucf_model'),
global_step=step)
print("done")
def train():
"""Train CIFAR-10 for a number of steps."""
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Get images and labels for CIFAR-10.
images, labels = cifar10.distorted_inputs()
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
print(labels)
print(logits)
# Calculate loss.
loss = cifar10.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = cifar10.train(loss, global_step)
# Create a saver.
saver = tf.train.Saver(tf.all_variables())
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.contrib.deprecated.merge_all_summaries()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
# summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
# graph_def=sess.graph_def)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
_, loss_value = sess.run([train_op, loss])
duration = time.time() - start_time
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
if step % 10 == 0:
num_examples_per_step = FLAGS.batch_size
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), step, loss_value,
examples_per_sec, sec_per_batch))
printfile(loss_value)
# Save the model checkpoint periodically.
if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
if step % 1000 == 0:
cifar10_eval.evaluate()
def cleanup(days, project, concurrency, silent, model, router, timed):
"""Delete a portion of trailing data based on creation date.
All data that is older than `--days` will be deleted. The default for
this is 30 days. In the default setting all projects will be truncated
but if you have a specific project you want to limit this to this can be
done with the `--project` flag which accepts a project ID or a string
with the form `org/project` where both are slugs.
"""
if concurrency < 1:
click.echo('Error: Minimum concurrency is 1', err=True)
raise click.Abort()
os.environ['_SENTRY_CLEANUP'] = '1'
# Make sure we fork off multiprocessing pool
# before we import or configure the app
from multiprocessing import Process, JoinableQueue as Queue
pool = []
task_queue = Queue(1000)
for _ in xrange(concurrency):
p = Process(target=multiprocess_worker, args=(task_queue, ))
p.daemon = True
p.start()
pool.append(p)
from sentry.runner import configure
configure()
from django.db import router as db_router
from sentry.app import nodestore
from sentry.db.deletion import BulkDeleteQuery
from sentry import models
if timed:
import time
from sentry.utils import metrics
start_time = time.time()
# list of models which this query is restricted to
model_list = {m.lower() for m in model}
def is_filtered(model):
if router is not None and db_router.db_for_write(model) != router:
return True
if not model_list:
return False
return model.__name__.lower() not in model_list
# Deletions that use `BulkDeleteQuery` (and don't need to worry about child relations)
# (model, datetime_field, order_by)
BULK_QUERY_DELETES = [
(models.EventMapping, 'date_added', '-date_added'),
(models.EventAttachment, 'date_added', None),
(models.UserReport, 'date_added', None),
(models.GroupEmailThread, 'date', None),
(models.GroupRuleStatus, 'date_added', None),
] + EXTRA_BULK_QUERY_DELETES
# Deletions that use the `deletions` code path (which handles their child relations)
# (model, datetime_field, order_by)
DELETES = (
(models.Event, 'datetime', 'datetime'),
(models.Group, 'last_seen', 'last_seen'),
)
if not silent:
click.echo('Removing expired values for LostPasswordHash')
if is_filtered(models.LostPasswordHash):
if not silent:
click.echo('>> Skipping LostPasswordHash')
else:
models.LostPasswordHash.objects.filter(date_added__lte=timezone.now() -
timedelta(hours=48)).delete()
if is_filtered(models.OrganizationMember) and not silent:
click.echo('>> Skipping OrganizationMember')
else:
click.echo('Removing expired values for OrganizationMember')
expired_threshold = timezone.now() - timedelta(days=days)
models.OrganizationMember.delete_expired(expired_threshold)
for model in [models.ApiGrant, models.ApiToken]:
if not silent:
click.echo(u'Removing expired values for {}'.format(
model.__name__))
if is_filtered(model):
if not silent:
click.echo(u'>> Skipping {}'.format(model.__name__))
else:
model.objects.filter(expires_at__lt=(
timezone.now() -
timedelta(days=API_TOKEN_TTL_IN_DAYS)), ).delete()
project_id = None
if project:
click.echo(
"Bulk NodeStore deletion not available for project selection",
err=True)
project_id = get_project(project)
if project_id is None:
click.echo('Error: Project not found', err=True)
raise click.Abort()
else:
if not silent:
click.echo("Removing old NodeStore values")
cutoff = timezone.now() - timedelta(days=days)
try:
nodestore.cleanup(cutoff)
except NotImplementedError:
click.echo("NodeStore backend does not support cleanup operation",
err=True)
for bqd in BULK_QUERY_DELETES:
if len(bqd) == 4:
model, dtfield, order_by, chunk_size = bqd
else:
chunk_size = 10000
model, dtfield, order_by = bqd
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
))
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
).execute(chunk_size=chunk_size)
for model, dtfield, order_by in DELETES:
if not silent:
click.echo(
u"Removing {model} for days={days} project={project}".format(
model=model.__name__,
days=days,
project=project or '*',
))
if is_filtered(model):
if not silent:
click.echo('>> Skipping %s' % model.__name__)
else:
imp = '.'.join((model.__module__, model.__name__))
q = BulkDeleteQuery(
model=model,
dtfield=dtfield,
days=days,
project_id=project_id,
order_by=order_by,
)
for chunk in q.iterator(chunk_size=100):
task_queue.put((imp, chunk))
task_queue.join()
# Clean up FileBlob instances which are no longer used and aren't super
# recent (as there could be a race between blob creation and reference)
if not silent:
click.echo("Cleaning up unused FileBlob references")
if is_filtered(models.FileBlob):
if not silent:
click.echo('>> Skipping FileBlob')
else:
cleanup_unused_files(silent)
# Shut down our pool
for _ in pool:
task_queue.put(_STOP_WORKER)
# And wait for it to drain
for p in pool:
p.join()
if timed:
duration = int(time.time() - start_time)
metrics.timing('cleanup.duration', duration, instance=router)
click.echo("Clean up took %s second(s)." % duration)
def train():
"""Train SqueezeDet model"""
assert FLAGS.dataset == 'KITTI', \
'Currently only support KITTI dataset'
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
with tf.Graph().as_default():
assert FLAGS.net == 'vgg16' or FLAGS.net == 'resnet50' \
or FLAGS.net == 'squeezeDet' or FLAGS.net == 'squeezeDet+', \
'Selected neural net architecture not supported: {}'.format(FLAGS.net)
if FLAGS.net == 'vgg16':
mc = kitti_vgg16_config()
mc.IS_TRAINING = True
mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
model = VGG16ConvDet(mc)
elif FLAGS.net == 'resnet50':
mc = kitti_res50_config()
mc.IS_TRAINING = True
mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
model = ResNet50ConvDet(mc)
elif FLAGS.net == 'squeezeDet':
mc = kitti_squeezeDet_config()
mc.IS_TRAINING = True
mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
model = SqueezeDet(mc)
elif FLAGS.net == 'squeezeDet+':
mc = kitti_squeezeDetPlus_config()
mc.IS_TRAINING = True
mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
model = SqueezeDetPlus(mc)
imdb = kitti(FLAGS.image_set, FLAGS.data_path, mc)
# save model size, flops, activations by layers
with open(os.path.join(FLAGS.train_dir, 'model_metrics.txt'),
'w') as f:
f.write('Number of parameter by layer:\n')
count = 0
for c in model.model_size_counter:
f.write('\t{}: {}\n'.format(c[0], c[1]))
count += c[1]
f.write('\ttotal: {}\n'.format(count))
count = 0
f.write('\nActivation size by layer:\n')
for c in model.activation_counter:
f.write('\t{}: {}\n'.format(c[0], c[1]))
count += c[1]
f.write('\ttotal: {}\n'.format(count))
count = 0
f.write('\nNumber of flops by layer:\n')
for c in model.flop_counter:
f.write('\t{}: {}\n'.format(c[0], c[1]))
count += c[1]
f.write('\ttotal: {}\n'.format(count))
f.close()
print('Model statistics saved to {}.'.format(
os.path.join(FLAGS.train_dir, 'model_metrics.txt')))
def _load_data(load_to_placeholder=True):
# read batch input
image_per_batch, label_per_batch, box_delta_per_batch, aidx_per_batch, \
bbox_per_batch = imdb.read_batch()
label_indices, bbox_indices, box_delta_values, mask_indices, box_values, \
= [], [], [], [], []
aidx_set = set()
num_discarded_labels = 0
num_labels = 0
for i in range(len(label_per_batch)): # batch_size
for j in range(len(
label_per_batch[i])): # number of annotations
num_labels += 1
if (i, aidx_per_batch[i][j]) not in aidx_set:
aidx_set.add((i, aidx_per_batch[i][j]))
label_indices.append(
[i, aidx_per_batch[i][j], label_per_batch[i][j]])
mask_indices.append([i, aidx_per_batch[i][j]])
bbox_indices.extend([[i, aidx_per_batch[i][j], k]
for k in range(4)])
box_delta_values.extend(box_delta_per_batch[i][j])
box_values.extend(bbox_per_batch[i][j])
else:
num_discarded_labels += 1
if mc.DEBUG_MODE:
print(
'Warning: Discarded {}/({}) labels that are assigned to the same '
'anchor'.format(num_discarded_labels, num_labels))
if load_to_placeholder:
image_input = model.ph_image_input
input_mask = model.ph_input_mask
box_delta_input = model.ph_box_delta_input
box_input = model.ph_box_input
labels = model.ph_labels
else:
image_input = model.image_input
input_mask = model.input_mask
box_delta_input = model.box_delta_input
box_input = model.box_input
labels = model.labels
feed_dict = {
image_input:
image_per_batch,
input_mask:
np.reshape(
sparse_to_dense(mask_indices, [mc.BATCH_SIZE, mc.ANCHORS],
[1.0] * len(mask_indices)),
[mc.BATCH_SIZE, mc.ANCHORS, 1]),
box_delta_input:
sparse_to_dense(bbox_indices, [mc.BATCH_SIZE, mc.ANCHORS, 4],
box_delta_values),
box_input:
sparse_to_dense(bbox_indices, [mc.BATCH_SIZE, mc.ANCHORS, 4],
box_values),
labels:
sparse_to_dense(label_indices,
[mc.BATCH_SIZE, mc.ANCHORS, mc.CLASSES],
[1.0] * len(label_indices)),
}
return feed_dict, image_per_batch, label_per_batch, bbox_per_batch
def _enqueue(sess, coord):
try:
while not coord.should_stop():
feed_dict, _, _, _ = _load_data()
sess.run(model.enqueue_op, feed_dict=feed_dict)
if mc.DEBUG_MODE:
print("added to the queue")
if mc.DEBUG_MODE:
print("Finished enqueue")
except Exception:
# coord.request_stop(e)
pass
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
saver = tf.train.Saver(tf.global_variables())
summary_op = tf.summary.merge_all()
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
init = tf.global_variables_initializer()
sess.run(init)
coord = tf.train.Coordinator()
if mc.NUM_THREAD > 0:
enq_threads = []
for _ in range(mc.NUM_THREAD):
enq_thread = threading.Thread(target=_enqueue,
args=[sess, coord])
# enq_thread.isDaemon()
enq_thread.start()
enq_threads.append(enq_thread)
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
run_options = tf.RunOptions(timeout_in_ms=60000)
# try:
for step in xrange(FLAGS.max_steps):
if coord.should_stop():
sess.run(model.FIFOQueue.close(cancel_pending_enqueues=True))
coord.request_stop()
coord.join(threads)
break
start_time = time.time()
if step % FLAGS.summary_step == 0:
feed_dict, image_per_batch, label_per_batch, bbox_per_batch = \
_load_data(load_to_placeholder=False)
op_list = [
model.train_op, model.loss, summary_op, model.det_boxes,
model.det_probs, model.det_class, model.conf_loss,
model.bbox_loss, model.class_loss
]
_, loss_value, summary_str, det_boxes, det_probs, det_class, \
conf_loss, bbox_loss, class_loss = sess.run(
op_list, feed_dict=feed_dict)
_viz_prediction_result(model, image_per_batch, bbox_per_batch,
label_per_batch, det_boxes, det_class,
det_probs)
image_per_batch = bgr_to_rgb(image_per_batch)
viz_summary = sess.run(
model.viz_op,
feed_dict={model.image_to_show: image_per_batch})
summary_writer.add_summary(summary_str, step)
summary_writer.add_summary(viz_summary, step)
summary_writer.flush()
print('conf_loss: {}, bbox_loss: {}, class_loss: {}'.format(
conf_loss, bbox_loss, class_loss))
else:
if mc.NUM_THREAD > 0:
_, loss_value, conf_loss, bbox_loss, class_loss = sess.run(
[
model.train_op, model.loss, model.conf_loss,
model.bbox_loss, model.class_loss
],
options=run_options)
else:
feed_dict, _, _, _ = _load_data(load_to_placeholder=False)
_, loss_value, conf_loss, bbox_loss, class_loss = sess.run(
[
model.train_op, model.loss, model.conf_loss,
model.bbox_loss, model.class_loss
],
feed_dict=feed_dict)
duration = time.time() - start_time
assert not np.isnan(loss_value), \
'Model diverged. Total loss: {}, conf_loss: {}, bbox_loss: {}, ' \
'class_loss: {}'.format(loss_value, conf_loss, bbox_loss, class_loss)
if step % 10 == 0:
num_images_per_step = mc.BATCH_SIZE
images_per_sec = num_images_per_step / duration
sec_per_batch = float(duration)
format_str = (
'%s: step %d, loss = %.2f (%.1f images/sec; %.3f '
'sec/batch)')
print(format_str % (datetime.now(), step, loss_value,
images_per_sec, sec_per_batch))
sys.stdout.flush()
# Save the model checkpoint periodically.
if step % FLAGS.checkpoint_step == 0 or (step +
1) == FLAGS.max_steps:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
def __del__(self):
for i in xrange(len(self.files)):
self.close(i, delete=True)
def read_annot(filepath, orig_ids=False):
"""Read in a Freesurfer annotation from a .annot file.
Parameters
----------
filepath : str
Path to annotation file.
orig_ids : bool
Whether to return the vertex ids as stored in the annotation
file or the positional colortable ids. With orig_ids=False
vertices with no id have an id set to -1.
Returns
-------
labels : ndarray, shape (n_vertices,)
Annotation id at each vertex. If a vertex does not belong
to any label and orig_ids=False, its id will be set to -1.
ctab : ndarray, shape (n_labels, 5)
RGBA + label id colortable array.
names : list of str
The names of the labels. The length of the list is n_labels.
"""
with open(filepath, "rb") as fobj:
dt = ">i4"
vnum = np.fromfile(fobj, dt, 1)[0]
data = np.fromfile(fobj, dt, vnum * 2).reshape(vnum, 2)
labels = data[:, 1]
ctab_exists = np.fromfile(fobj, dt, 1)[0]
if not ctab_exists:
raise Exception('Color table not found in annotation file')
n_entries = np.fromfile(fobj, dt, 1)[0]
if n_entries > 0:
length = np.fromfile(fobj, dt, 1)[0]
orig_tab = np.fromfile(fobj, '>c', length)
orig_tab = orig_tab[:-1]
names = list()
ctab = np.zeros((n_entries, 5), np.int)
for i in xrange(n_entries):
name_length = np.fromfile(fobj, dt, 1)[0]
name = np.fromfile(fobj, "|S%d" % name_length, 1)[0]
names.append(name)
ctab[i, :4] = np.fromfile(fobj, dt, 4)
ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
ctab[i, 2] * (2 ** 16) +
ctab[i, 3] * (2 ** 24))
else:
ctab_version = -n_entries
if ctab_version != 2:
raise Exception('Color table version not supported')
n_entries = np.fromfile(fobj, dt, 1)[0]
ctab = np.zeros((n_entries, 5), np.int)
length = np.fromfile(fobj, dt, 1)[0]
np.fromfile(fobj, "|S%d" % length, 1)[0] # Orig table path
entries_to_read = np.fromfile(fobj, dt, 1)[0]
names = list()
for i in xrange(entries_to_read):
np.fromfile(fobj, dt, 1)[0] # Structure
name_length = np.fromfile(fobj, dt, 1)[0]
name = np.fromfile(fobj, "|S%d" % name_length, 1)[0]
names.append(name)
ctab[i, :4] = np.fromfile(fobj, dt, 4)
ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
ctab[i, 2] * (2 ** 16))
ctab[:, 3] = 255
labels = labels.astype(np.int)
if not orig_ids:
ord = np.argsort(ctab[:, -1])
mask = labels != 0
labels[~mask] = -1
labels[mask] = ord[np.searchsorted(ctab[ord, -1], labels[mask])]
return labels, ctab, names
def timer(label, iters=30000):
start = time.time()
yield xrange(iters)
end = time.time()
t = (end - start) * 1e6 / iters
print("%-40s took %.2fus (%d iterations)" % (label, t, iters))
def forecast(self, y, horizon=0, u=None):
"""
Calculate an one-step-ahead forecast.
:param y: output time series
:param horizon: number of predictions after y[T_max]
:param u: external input time series
:return: predicted time series as array
"""
p = self.A.shape[1]
a, b = self.A.shape[0], self.B.shape[0]
c, m = self.C.shape[0], self.C.shape[2]
u = u if u is not None else np.zeros((c, m))
y = utils.atleast_2d(y)
sampleT = y.shape[0]
predictT = sampleT + horizon
# diagonalize with respect to matrix of B's leading coefficients
B0inv = linalg.inv(self.B[0, :, :])
A = np.tensordot(self.A, B0inv, axes=1)
B = np.tensordot(self.B, B0inv, axes=1)
if c != 0:
C = np.einsum('ijk,kl', self.C, B0inv)
else:
C = np.zeros((c, p, m))
# calculate directly the residual ...
res = -np.dot(self._prep_trend(sampleT, p)[:sampleT, ...], B0inv)
# and perform prediction
for t in xrange(sampleT):
la, lb, lc = min(a - 1, t), min(b - 1, t), min(c - 1, t)
ba, bb, bc = max(0, t - la), max(0, t - lb), max(0, t - lc)
res[t, :] += np.einsum('ikj,ij', A[la::-1, ...], y[ba:t + 1, :])
if b != 0:
res[t, :] -= np.einsum('ikj,ij', B[lb:0:-1, ...], res[bb:t, :])
if c != 0:
res[t, :] -= np.einsum('ikj,ij', C[lc::-1, ...],
u[bc:t + 1, :])
pred = np.zeros((predictT, p))
pred[:sampleT, :] = y[:sampleT, :] - np.dot(res, B[0, :, :])
if predictT > sampleT:
A0inv = linalg.inv(self.A[0, :, :])
A = np.tensordot(self.A, A0inv, axes=1)
B = np.tensordot(self.B, A0inv, axes=1)
if c != 0:
C = np.einsum('ijk,kl', self.C, A0inv)
else:
C = np.zeros((c, p, m))
pred[sampleT:, :] = np.dot(self._prep_trend(horizon, p, sampleT),
A0inv)
# perform prediction for horizon period
for t in xrange(sampleT, predictT):
for l in xrange(1, a):
if t - l < sampleT:
pred[t, :] -= np.dot(A[l, :, :], y[t - l, :])
else:
pred[t, :] -= np.dot(A[l, :, :], pred[t - l, :])
for l in xrange(b):
if t - l < sampleT:
pred[t, :] += np.dot(B[l, :, :], res[t - l, :])
for l in xrange(c):
pred[t, :] += np.dot(C[l, :, :], u[t - l, :])
return pred
def collect_general_canonical_repn(exp, idMap, compute_values):
# global temp_const
# global temp_var
# global temp_nonl
temp_const = { 0: {None:0.0} }
temp_var = { 1: {GeneralCanonicalRepn({None:1}):1.0} }
temp_nonl = { None: None }
exp_type = type(exp)
#
# Constant
#
if exp.is_fixed():
if compute_values:
temp_const[0][None] = value(exp)
else:
temp_const[0][None] = exp
return temp_const
#
# Expression
#
elif exp.is_expression():
#
# Sum
#
if exp_type is expr._SumExpression:
if exp._const != 0.0:
repn = { 0: {None:exp._const} }
else:
repn = {}
for i in xrange(len(exp._args)):
repn = repn_add(
repn,
collect_general_canonical_repn(exp._args[i],
idMap,
compute_values),
coef=exp._coef[i])
return repn
#
# Product
#
elif exp_type is expr._ProductExpression:
#
# Iterate through the denominator. If they aren't all
# constants, then simply return this expression.
#
denom=1.0
for e in exp._denominator:
if e.is_fixed():
denom *= e()
else:
temp_nonl[None] = exp
return temp_nonl
if denom == 0.0:
print("Divide-by-zero error - offending sub-expression:")
e.pprint()
raise ZeroDivisionError
#
# OK, the denominator is a constant.
#
repn = { 0: {None:exp._coef / denom} }
for e in exp._numerator:
repn = repn_mult(
repn,
collect_general_canonical_repn(e,
idMap,
compute_values))
return repn
#
# Power Expression
#
elif exp_type is expr._PowExpression:
if exp.polynomial_degree() is None:
raise TypeError("Unsupported general power expression: "
+str(exp._args))
# If this is of the form EXPR**1, we can just get the
# representation of EXPR
if exp._args[1] == 1:
return collect_general_canonical_repn(exp._args[0],
idMap,
compute_values)
# The only other way to get a polynomial expression is if
# exp=EXPR**p where p is fixed a nonnegative integer. We
# can expand this expression and generate a canonical
# representation from there. If p=0, this expression is
# constant (and is processed by the is_fixed code above
# NOTE: There is no check for 0**0
return collect_general_canonical_repn(
reduce( lambda x,y: x*y, [exp._args[0]]*int(value(exp._args[1])), 1.0 ),
idMap,
compute_values)
elif exp_type is expr.Expr_if:
if exp._if.is_fixed():
if exp._if():
return collect_general_canonical_repn(exp._then,
idMap,
compute_values)
else:
return collect_general_canonical_repn(exp._else,
idMap,
compute_values)
else:
temp_nonl[None] = exp
return temp_nonl
#
# Expression (the component)
# (faster check)
elif isinstance(exp, _ExpressionData):
return collect_general_canonical_repn(exp.expr,
idMap,
compute_values)
#
# ERROR
#
else:
raise ValueError("Unsupported expression type: "+str(exp))
#
# Variable
#
elif (exp.__class__ is _GeneralVarData) or isinstance(exp, _VarData):
id_ = id(exp)
if id_ in idMap[None]:
key = idMap[None][id_]
else:
key = len(idMap) - 1
idMap[None][id_] = key
idMap[key] = exp
temp_var = { -1: {key:exp}, 1: {GeneralCanonicalRepn({key:1}):1.0} }
return temp_var
#
# Connector
#
elif exp_type is _ConnectorValue or exp.type() is Connector:
# Silently omit constraint... The ConnectorExpander should
# expand this constraint into indvidual constraints that
# reference "real" variables.
return {}
#
# ERROR
#
else:
raise ValueError("Unexpected expression (type %s): " %
( type(exp).__name__, str(exp) ))
def from_keypoint_image(image, if_not_found_coords={"x": -1, "y": -1},
threshold=1, nb_channels=None):
"""Convert ``to_keypoint_image()`` outputs to ``KeypointsOnImage``.
This is the inverse of :func:`KeypointsOnImage.to_keypoint_image`.
Parameters
----------
image : (H,W,N) ndarray
The keypoints image. N is the number of keypoints.
if_not_found_coords : tuple or list or dict or None, optional
Coordinates to use for keypoints that cannot be found in `image`.
* If this is a ``list``/``tuple``, it must contain two ``int``
values.
* If it is a ``dict``, it must contain the keys ``x`` and
``y`` with each containing one ``int`` value.
* If this is ``None``, then the keypoint will not be added to the
final :class:`KeypointsOnImage` object.
threshold : int, optional
The search for keypoints works by searching for the argmax in
each channel. This parameters contains the minimum value that
the max must have in order to be viewed as a keypoint.
nb_channels : None or int, optional
Number of channels of the image on which the keypoints are placed.
Some keypoint augmenters require that information.
If set to ``None``, the keypoint's shape will be set
to ``(height, width)``, otherwise ``(height, width, nb_channels)``.
Returns
-------
imgaug.augmentables.kps.KeypointsOnImage
The extracted keypoints.
"""
# pylint: disable=dangerous-default-value
assert image.ndim == 3, (
"Expected 'image' to have three dimensions, "
"got %d with shape %s instead." % (image.ndim, image.shape))
height, width, nb_keypoints = image.shape
drop_if_not_found = False
if if_not_found_coords is None:
drop_if_not_found = True
if_not_found_x = -1
if_not_found_y = -1
elif isinstance(if_not_found_coords, (tuple, list)):
assert len(if_not_found_coords) == 2, (
"Expected tuple 'if_not_found_coords' to contain exactly two "
"values, got %d values." % (len(if_not_found_coords),))
if_not_found_x = if_not_found_coords[0]
if_not_found_y = if_not_found_coords[1]
elif isinstance(if_not_found_coords, dict):
if_not_found_x = if_not_found_coords["x"]
if_not_found_y = if_not_found_coords["y"]
else:
raise Exception(
"Expected if_not_found_coords to be None or tuple or list "
"or dict, got %s." % (type(if_not_found_coords),))
keypoints = []
for i in sm.xrange(nb_keypoints):
maxidx_flat = np.argmax(image[..., i])
maxidx_ndim = np.unravel_index(maxidx_flat, (height, width))
found = (image[maxidx_ndim[0], maxidx_ndim[1], i] >= threshold)
if found:
x = maxidx_ndim[1] + 0.5
y = maxidx_ndim[0] + 0.5
keypoints.append(Keypoint(x=x, y=y))
else:
if drop_if_not_found:
# dont add the keypoint to the result list, i.e. drop it
pass
else:
keypoints.append(Keypoint(x=if_not_found_x,
y=if_not_found_y))
out_shape = (height, width)
if nb_channels is not None:
out_shape += (nb_channels,)
return KeypointsOnImage(keypoints, shape=out_shape)
normalized_embeddings,
transpose_b=True)
# Add variable initializer.
init = tf.global_variables_initializer()
# Step 5: Begin training.
num_steps = 20001
with tf.Session(graph=graph) as session:
# We must initialize all variables before we use them.
init.run()
print('Initialized')
average_loss = 0
for step in xrange(num_steps):
batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
_, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
average_loss += loss_val
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000 batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
def train(args):
"""Train different architectures for a number of epochs."""
with tf.Graph().as_default(), tf.device('/cpu:0'):
# Read data from disk
images, labels = data_loader.read_inputs(True, args)
#epoch number
epoch_number = tf.get_variable('epoch_number', [], dtype= tf.int32, initializer= tf.constant_initializer(0), trainable= False)
# Decay the learning rate
lr = tf.train.piecewise_constant(epoch_number, args.LR_steps, args.LR_values, name= 'LearningRate')
# Weight Decay policy
wd = tf.train.piecewise_constant(epoch_number, args.WD_steps, args.WD_values, name= 'WeightDecay')
# transfer mode 1 means using the model as a feature extractor, so we don't need batch norm updates and dropout
is_training= not args.transfer_mode[0]==1
# Create an optimizer that performs gradient descent.
opt = tf.train.MomentumOptimizer(lr, 0.9)
# Calculate the gradients for each model tower.
tower_grads = []
# it is only needed for transfer mode 3
tower_auxgrads = []
with tf.variable_scope(tf.get_variable_scope()):
for i in xrange(args.num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('Tower_%d' % i) as scope:
# Calculate the loss for one tower. This function
# constructs the entire model but shares the variables across
# all towers.
logits = arch.get_model(images, wd, is_training, args)
# Top-1 accuracy
top1acc = tf.reduce_mean(
tf.cast(tf.nn.in_top_k(logits, labels, 1), tf.float32))
# Top-n accuracy
# topnacc = tf.reduce_mean(
# tf.cast(tf.nn.in_top_k(logits, labels, args.top_n), tf.float32))
# Build the portion of the Graph calculating the losses. Note that we will
# assemble the total_loss using a custom function below.
cross_entropy_mean = loss(logits, labels)
# Get all the regularization lesses and add them
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
reg_loss = tf.add_n(regularization_losses)
#Add a tensorboard summary
tf.summary.scalar('Regularization Loss', reg_loss)
# Compute the total loss (cross entropy loss + regularization loss)
total_loss = tf.add(cross_entropy_mean, reg_loss)
# Attach a scalar summary for the total loss and top-1 and top-5 accuracies
tf.summary.scalar('Total Loss', total_loss)
tf.summary.scalar('Top-1 Accuracy', top1acc)
# tf.summary.scalar('Top-'+str(args.top_n)+' Accuracy', topnacc)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
# Retain the summaries from the final tower.
summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)
# Gather batch normaliziation update operations
batchnorm_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope)
# Calculate the gradients for the batch of data on this tower.
if args.transfer_mode[0]== 3:
# compute gradients for the last layer only
grads = opt.compute_gradients(total_loss, var_list= tf.get_collection(tf.GraphKeys.VARIABLES, scope='output'))
# compute gradients for all layers
auxgrads = opt.compute_gradients(total_loss)
tower_auxgrads.append(auxgrads)
elif args.transfer_mode[0]==1:
grads = opt.compute_gradients(total_loss,var_list= tf.get_collection(tf.GraphKeys.VARIABLES, scope='output'))
else:
grads = opt.compute_gradients(total_loss)
# Keep track of the gradients across all towers.
tower_grads.append(grads)
# We must calculate the mean of each gradient. Note that this is the
# synchronization point across all towers.
grads = average_gradients(tower_grads)
# average all gradients for transfer mode 3
if args.transfer_mode[0]== 3:
auxgrads = average_gradients(tower_auxgrads)
# Add a summary to track the learning rate and weight decay
summaries.append(tf.summary.scalar('learning_rate', lr))
summaries.append(tf.summary.scalar('weight_decay', wd))
# Setup the train operation
if args.transfer_mode[0]==3:
train_op = tf.cond(tf.less(epoch_number,args.transfer_mode[1]),
lambda: tf.group(opt.apply_gradients(grads),*batchnorm_updates), lambda: tf.group(opt.apply_gradients(auxgrads),*batchnorm_updates))
elif args.transfer_mode[0]==1:
train_op = opt.apply_gradients(grads)
else:
batchnorm_updates_op = tf.group(*batchnorm_updates)
train_op = tf.group(opt.apply_gradients(grads), batchnorm_updates_op)
# a loader for loading the pretrained model (it does not load the last layer)
if args.retrain_from is not None:
if args.transfer_mode[0]==0:
pretrained_loader = tf.train.Saver()
else:
pretrained_loader = tf.train.Saver(var_list= exclude())
# Create a saver.
saver = tf.train.Saver(tf.global_variables(), max_to_keep= args.max_to_keep)
# Build the summary operation from the last tower summaries.
summary_op = tf.summary.merge_all()
# Build an initialization operation to run below.
init = tf.global_variables_initializer()
# Logging the runtime information if requested
if args.log_debug_info:
run_options = tf.RunOptions(trace_level= tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
else:
run_options = None
run_metadata = None
# Creating a session to run the built graph
tf_config = tf.ConfigProto()
tf_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf_config.log_device_placement = True
tf_config.allow_soft_placement = True
sess = tf.Session(config=tf_config)
sess.run(init)
# Continue training from a saved snapshot, load a pre-trained model
if args.retrain_from is not None:
ckpt = tf.train.get_checkpoint_state(args.retrain_from)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
pretrained_loader.restore(sess, ckpt.model_checkpoint_path)
else:
return
# Start the queue runners.
tf.train.start_queue_runners(sess= sess)
# Setup a summary writer
summary_writer = tf.summary.FileWriter(args.log_dir, sess.graph)
# Set the start epoch number
start_epoch = sess.run(epoch_number + 1)
# The main training loop
for epoch in xrange(start_epoch, start_epoch + args.num_epochs):
# update epoch_number
sess.run(epoch_number.assign(epoch))
# Trainig batches
for step in xrange(args.num_batches):
start_time = time.time()
_, loss_value, top1_accuracy = sess.run(
[train_op, cross_entropy_mean, top1acc], options= run_options, run_metadata= run_metadata)
duration = time.time() - start_time
# Check for errors
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
# Logging and writing tensorboard summaries
if step % 10 == 0:
num_examples_per_step = args.chunked_batch_size * args.num_gpus
examples_per_sec = num_examples_per_step / duration
sec_per_batch = duration / args.num_gpus
format_str = ('%s: epoch %d, step %d, loss = %.2f, Top-1 = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), epoch, step, loss_value, top1_accuracy,
examples_per_sec, sec_per_batch))
sys.stdout.flush()
if step % 2000 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(
summary_str, args.num_batches * epoch + step)
if args.log_debug_info:
summary_writer.add_run_metadata(
run_metadata, 'epoch%d step%d' % (epoch, step))
# Save the model checkpoint periodically after each training epoch
checkpoint_path = os.path.join(args.log_dir, args.snapshot_prefix)
print("checkpoint_path-----------",checkpoint_path)
saver.save(sess, checkpoint_path, global_step= epoch)
def from_distance_maps(distance_maps, inverted=False,
if_not_found_coords={"x": -1, "y": -1},
threshold=None, nb_channels=None):
"""Convert outputs of ``to_distance_maps()`` to ``KeypointsOnImage``.
This is the inverse of :func:`KeypointsOnImage.to_distance_maps`.
Parameters
----------
distance_maps : (H,W,N) ndarray
The distance maps. ``N`` is the number of keypoints.
inverted : bool, optional
Whether the given distance maps were generated in inverted mode
(i.e. :func:`KeypointsOnImage.to_distance_maps` was called with
``inverted=True``) or in non-inverted mode.
if_not_found_coords : tuple or list or dict or None, optional
Coordinates to use for keypoints that cannot be found
in `distance_maps`.
* If this is a ``list``/``tuple``, it must contain two ``int``
values.
* If it is a ``dict``, it must contain the keys ``x`` and
``y`` with each containing one ``int`` value.
* If this is ``None``, then the keypoint will not be added to the
final :class:`KeypointsOnImage` object.
threshold : float, optional
The search for keypoints works by searching for the
argmin (non-inverted) or argmax (inverted) in each channel. This
parameters contains the maximum (non-inverted) or
minimum (inverted) value to accept in order to view a hit as a
keypoint. Use ``None`` to use no min/max.
nb_channels : None or int, optional
Number of channels of the image on which the keypoints are placed.
Some keypoint augmenters require that information.
If set to ``None``, the keypoint's shape will be set
to ``(height, width)``, otherwise ``(height, width, nb_channels)``.
Returns
-------
imgaug.augmentables.kps.KeypointsOnImage
The extracted keypoints.
"""
# pylint: disable=dangerous-default-value
assert distance_maps.ndim == 3, (
"Expected three-dimensional input, got %d dimensions and "
"shape %s." % (distance_maps.ndim, distance_maps.shape))
height, width, nb_keypoints = distance_maps.shape
drop_if_not_found = False
if if_not_found_coords is None:
drop_if_not_found = True
if_not_found_x = -1
if_not_found_y = -1
elif isinstance(if_not_found_coords, (tuple, list)):
assert len(if_not_found_coords) == 2, (
"Expected tuple/list 'if_not_found_coords' to contain "
"exactly two entries, got %d." % (len(if_not_found_coords),))
if_not_found_x = if_not_found_coords[0]
if_not_found_y = if_not_found_coords[1]
elif isinstance(if_not_found_coords, dict):
if_not_found_x = if_not_found_coords["x"]
if_not_found_y = if_not_found_coords["y"]
else:
raise Exception(
"Expected if_not_found_coords to be None or tuple or list or "
"dict, got %s." % (type(if_not_found_coords),))
keypoints = []
for i in sm.xrange(nb_keypoints):
# TODO introduce voting here among all distance values that have
# min/max values
if inverted:
hitidx_flat = np.argmax(distance_maps[..., i])
else:
hitidx_flat = np.argmin(distance_maps[..., i])
hitidx_ndim = np.unravel_index(hitidx_flat, (height, width))
if not inverted and threshold is not None:
found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i]
< threshold)
elif inverted and threshold is not None:
found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i]
>= threshold)
else:
found = True
if found:
keypoints.append(Keypoint(x=hitidx_ndim[1], y=hitidx_ndim[0]))
else:
if drop_if_not_found:
# dont add the keypoint to the result list, i.e. drop it
pass
else:
keypoints.append(Keypoint(x=if_not_found_x,
y=if_not_found_y))
out_shape = (height, width)
if nb_channels is not None:
out_shape += (nb_channels,)
return KeypointsOnImage(keypoints, shape=out_shape)
def ListsAreClose(self, result, expected, rtol, atol):
"""Tests closeness of two lists of floats."""
self.assertEqual(len(result), len(expected))
for i in xrange(len(result)):
self.assertAllClose(result[i], expected[i], rtol, atol)
def main(_):
# We want to see all the logging messages for this tutorial.
tf.logging.set_verbosity(tf.logging.INFO)
# Start a new TensorFlow session.
sess = tf.InteractiveSession()
# Begin by making sure we have the training data we need.
model_settings = models.prepare_model_settings(
len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))),
FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms,
FLAGS.window_stride_ms, FLAGS.dct_coefficient_count)
audio_processor = input_data.AudioProcessor(
FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage,
FLAGS.wanted_words.split(','), FLAGS.validation_percentage,
FLAGS.testing_percentage, model_settings)
fingerprint_size = model_settings['fingerprint_size']
label_count = model_settings['label_count']
time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
# Figure out the learning rates for each training phase.
# Since it's often effective to have high learning rates at the start of training,
# followed by lower levels towards the end, the number of steps and learning rates can be
# specified as comma-separated lists to define the rate at each stage.
# For example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
# will run 13,000 training loops in total, with a rate of 0.001 for the first
# 10,000, and 0.0001 for the final 3,000.
training_steps_list = list(
map(int, FLAGS.how_many_training_steps.split(',')))
learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))
if len(training_steps_list) != len(learning_rates_list):
raise Exception(
'--how_many_training_steps and --learning_rate must be equal length lists, '
'but are %d and %d long instead' %
(len(training_steps_list), len(learning_rates_list)))
fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size],
name='fingerprint_input')
logits, dropout_prob = models.create_model(fingerprint_input,
model_settings,
FLAGS.model_architecture,
is_training=True)
# Define loss and optimizer
ground_truth_input = tf.placeholder(tf.int64, [None],
name='groundtruth_input')
# Optionally we can add runtime checks to spot when NaNs or other symptoms of
# numerical errors start occurring during training.
control_dependencies = []
if FLAGS.check_nans:
checks = tf.add_check_numerics_ops()
control_dependencies = [checks]
# Create the back propagation and training evaluation machinery in the graph.
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels=ground_truth_input, logits=logits)
tf.summary.scalar('cross_entropy', cross_entropy_mean)
with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
learning_rate_input = tf.placeholder(tf.float32, [],
name='learning_rate_input')
train_step = tf.train.GradientDescentOptimizer(
learning_rate_input).minimize(cross_entropy_mean)
predicted_indices = tf.argmax(logits, 1)
correct_prediction = tf.equal(predicted_indices, ground_truth_input)
confusion_matrix = tf.confusion_matrix(ground_truth_input,
predicted_indices,
num_classes=label_count)
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', evaluation_step)
global_step = tf.train.get_or_create_global_step()
increment_global_step = tf.assign(global_step, global_step + 1)
saver = tf.train.Saver(tf.global_variables())
# Merge all the summaries and write them out to /tmp/retrain_logs (by default)
merged_summaries = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir +
'/validation')
tf.global_variables_initializer().run()
start_step = 1
if FLAGS.start_checkpoint:
models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
start_step = global_step.eval(session=sess)
tf.logging.info('Training from step: %d ', start_step)
# Save graph.pbtxt.
tf.train.write_graph(sess.graph_def, FLAGS.train_dir,
FLAGS.model_architecture + '.pbtxt')
# Save list of words.
with gfile.GFile(
os.path.join(FLAGS.train_dir,
FLAGS.model_architecture + '_labels.txt'), 'w') as f:
f.write('\n'.join(audio_processor.words_list))
# Training loop.
training_steps_max = np.sum(training_steps_list)
for training_step in xrange(start_step, training_steps_max + 1):
# Figure out what the current learning rate is.
training_steps_sum = 0
for i in range(len(training_steps_list)):
training_steps_sum += training_steps_list[i]
if training_step <= training_steps_sum:
learning_rate_value = learning_rates_list[i]
break
# Pull the audio samples we'll use for training.
train_fingerprints, train_ground_truth = audio_processor.get_data(
FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency,
FLAGS.background_volume, time_shift_samples, 'training', sess)
# Run the graph with this batch of training data.
train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
[
merged_summaries, evaluation_step, cross_entropy_mean,
train_step, increment_global_step
],
feed_dict={
fingerprint_input: train_fingerprints,
ground_truth_input: train_ground_truth,
learning_rate_input: learning_rate_value,
dropout_prob: 0.5
})
train_writer.add_summary(train_summary, training_step)
tf.logging.info(
'Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' %
(training_step, learning_rate_value, train_accuracy * 100,
cross_entropy_value))
# Output the confusion matrix and validation accuracy periodically
is_last_step = (training_step == training_steps_max)
if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step:
set_size = audio_processor.set_size('validation')
total_accuracy = 0
total_conf_matrix = None
for i in xrange(0, set_size, FLAGS.batch_size):
validation_fingerprints, validation_ground_truth = audio_processor.get_data(
FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0,
'validation', sess)
# Run a validation step and capture training summaries for TensorBoard with the `merged` op.
validation_summary, validation_accuracy, conf_matrix = sess.run(
[merged_summaries, evaluation_step, confusion_matrix],
feed_dict={
fingerprint_input: validation_fingerprints,
ground_truth_input: validation_ground_truth,
dropout_prob: 1.0
})
validation_writer.add_summary(validation_summary,
training_step)
batch_size = min(FLAGS.batch_size, set_size - i)
total_accuracy += (validation_accuracy * batch_size) / set_size
if total_conf_matrix is None:
total_conf_matrix = conf_matrix
else:
total_conf_matrix += conf_matrix
tf.logging.info('Confusion Matrix:\n %s' % total_conf_matrix)
tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' %
(training_step, total_accuracy * 100, set_size))
# Save the model checkpoint periodically.
if training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max:
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.model_architecture + '.ckpt')
tf.logging.info('Saving to "%s-%d"', checkpoint_path,
training_step)
saver.save(sess, checkpoint_path, global_step=training_step)
# Testing loop.
set_size = audio_processor.set_size('testing')
tf.logging.info('set_size=%d', set_size)
total_accuracy = 0
total_conf_matrix = None
for i in xrange(0, set_size, FLAGS.batch_size):
test_fingerprints, test_ground_truth = audio_processor.get_data(
FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess)
test_accuracy, conf_matrix = sess.run(
[evaluation_step, confusion_matrix],
feed_dict={
fingerprint_input: test_fingerprints,
ground_truth_input: test_ground_truth,
dropout_prob: 1.0
})
batch_size = min(FLAGS.batch_size, set_size - i)
total_accuracy += (test_accuracy * batch_size) / set_size
if total_conf_matrix is None:
total_conf_matrix = conf_matrix
else:
total_conf_matrix += conf_matrix
tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
tf.logging.info('Final test accuracy = %.1f%% (N=%d)' %
(total_accuracy * 100, set_size))
def refill_batches(batches, word2id, context_file, qn_file, ans_file,
batch_size, context_len, question_len, word_len,
discard_long):
"""
Adds more batches into the "batches" list.
Inputs:
batches: list to add batches to
word2id: dictionary mapping word (string) to word id (int)
context_file, qn_file, ans_file: paths to {train/dev}.{context/question/answer} data files
batch_size: int. how big to make the batches
context_len, question_len: max length of context and question respectively
discard_long: If True, discard any examples that are longer than context_len or question_len.
If False, truncate those exmaples instead.
"""
print "Refilling batches..."
tic = time.time()
examples = [
] # list of (qn_ids, context_ids, ans_span, ans_tokens) triples
context_line, qn_line, ans_line = context_file.readline(
), qn_file.readline(), ans_file.readline() # read the next line from each
while context_line and qn_line and ans_line: # while you haven't reached the end
# Convert tokens to word ids
context_tokens, context_ids, context_char_ids = sentence_to_token_ids(
context_line, word2id)
qn_tokens, qn_ids, qn_char_ids = sentence_to_token_ids(
qn_line, word2id)
ans_span = intstr_to_intlist(ans_line)
# read the next line from each file
context_line, qn_line, ans_line = context_file.readline(
), qn_file.readline(), ans_file.readline()
# get ans_tokens from ans_span
assert len(ans_span) == 2
if ans_span[1] < ans_span[0]:
print "Found an ill-formed gold span: start=%i end=%i" % (
ans_span[0], ans_span[1])
continue
ans_tokens = context_tokens[ans_span[0]:ans_span[1] +
1] # list of strings
# always truncate too long words
for context_char_id in context_char_ids:
if len(context_char_id) > word_len:
context_char_id = context_char_id[:word_len]
if len(context_char_id) < word_len:
context_char_id.extend([CHAR_PAD_ID] *
(word_len - len(context_char_id)))
context_char_ids_flat = [
item for sublist in context_char_ids for item in sublist
]
context_char_len = context_len * word_len
if len(context_char_ids_flat) > context_char_len:
if discard_long:
continue
else:
context_char_ids_flat = context_char_ids_flat[:
context_char_len]
# always truncate too long words
for qn_char_id in qn_char_ids:
if len(qn_char_id) > word_len:
context_char_id = context_char_id[:word_len]
if len(qn_char_id) < word_len:
qn_char_id.extend([CHAR_PAD_ID] * (word_len - len(qn_char_id)))
question_char_len = question_len * word_len
qn_char_ids_flat = [
item for sublist in qn_char_ids for item in sublist
]
if len(qn_char_ids_flat) > question_char_len:
if discard_long:
continue
else:
qn_char_ids_flat = qn_char_ids_flat[:question_char_len]
# discard or truncate too-long questions
if len(qn_ids) > question_len:
if discard_long:
continue
else: # truncate
qn_ids = qn_ids[:question_len]
# discard or truncate too-long contexts
if len(context_ids) > context_len:
if discard_long:
continue
else: # truncate
context_ids = context_ids[:context_len]
# add to examples
examples.append(
(context_ids, context_tokens, qn_ids, qn_tokens,
context_char_ids_flat, qn_char_ids_flat, ans_span, ans_tokens))
# stop refilling if you have 160 batches
if len(examples) == batch_size * 160:
break
# Once you've either got 160 batches or you've reached end of file:
# Sort by question length
# Note: if you sort by context length, then you'll have batches which contain the same context many times (because each context appears several times, with different questions)
examples = sorted(examples, key=lambda e: len(e[2]))
# Make into batches and append to the list batches
for batch_start in xrange(0, len(examples), batch_size):
# Note: each of these is a list length batch_size of lists of ints (except on last iter when it might be less than batch_size)
context_ids_batch, context_tokens_batch, qn_ids_batch, qn_tokens_batch, context_char_ids_batch, qn_char_ids_batch, ans_span_batch, ans_tokens_batch = zip(
*examples[batch_start:batch_start + batch_size])
batches.append((context_ids_batch, context_tokens_batch, qn_ids_batch,
qn_tokens_batch, context_char_ids_batch,
qn_char_ids_batch, ans_span_batch, ans_tokens_batch))
# shuffle the batches
random.shuffle(batches)
toc = time.time()
print "Refilling batches took %.2f seconds" % (toc - tic)
return
def train():
# prepare dataset
print("Preparing data in %s" % gConfig['working_directory'])
enc_train, dec_train, enc_dev, dec_dev, _, _ = data_utils.prepare_custom_data(gConfig['working_directory'],gConfig['train_enc'],gConfig['train_dec'],gConfig['test_enc'],gConfig['test_dec'],gConfig['enc_vocab_size'],gConfig['dec_vocab_size'])
# Only allocate 2/3 of the gpu memory to allow for running gpu-based predictions while training:
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.666)
config = tf.ConfigProto(gpu_options=gpu_options)
config.gpu_options.allocator_type = 'BFC'
with tf.Session(config=config) as sess:
# Create model.
print("Creating %d layers of %d units." % (gConfig['num_layers'], gConfig['layer_size']))
model = create_model(sess, False)
# Read data into buckets and compute their sizes.
print ("Reading development and training data (limit: %d)."
% gConfig['max_train_data_size'])
dev_set = read_data(enc_dev, dec_dev)
train_set = read_data(enc_train, dec_train, gConfig['max_train_data_size'])
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
train_total_size = float(sum(train_bucket_sizes))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))]
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = np.random.random_sample()
bucket_id = min([i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01])
# Get a batch and make a step.
start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id)
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / gConfig['steps_per_checkpoint']
loss += step_loss / gConfig['steps_per_checkpoint']
current_step += 1
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % gConfig['steps_per_checkpoint'] == 0:
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
print ("global step %d learning rate %.4f step-time %.2f perplexity "
"%.2f" % (model.global_step.eval(), model.learning_rate.eval(),
step_time, perplexity))
# Decrease learning rate if no improvement was seen over last 3 times.
if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(gConfig['working_directory'], "seq2seq.ckpt")
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
step_time, loss = 0.0, 0.0
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(_buckets)):
if len(dev_set[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
dev_set, bucket_id)
_, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, True)
eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx))
sys.stdout.flush()
def _GetGrads(grads, op):
"""Gets all gradients for op."""
if op in grads:
return grads[op]
else:
return [[] for _ in xrange(len(op.outputs))]
def _build(self):
"""Builds a model."""
self._blocks = []
batch_norm_momentum = self._global_params.batch_norm_momentum
batch_norm_epsilon = self._global_params.batch_norm_epsilon
if self._global_params.data_format == 'channels_first':
channel_axis = 1
self._spatial_dims = [2, 3]
else:
channel_axis = -1
self._spatial_dims = [1, 2]
# Stem part.
self._conv_stem = tf.layers.Conv2D(
filters=round_filters(32, self._global_params),
kernel_size=[3, 3],
strides=[2, 2],
kernel_initializer=conv_kernel_initializer,
padding='same',
data_format=self._global_params.data_format,
use_bias=False)
self._bn0 = self._batch_norm(axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon)
# Builds blocks.
for block_args in self._blocks_args:
assert block_args.num_repeat > 0
assert block_args.super_pixel in [0, 1, 2]
# Update block input and output filters based on depth multiplier.
input_filters = round_filters(block_args.input_filters,
self._global_params)
output_filters = round_filters(block_args.output_filters,
self._global_params)
kernel_size = block_args.kernel_size
block_args = block_args._replace(input_filters=input_filters,
output_filters=output_filters,
num_repeat=round_repeats(
block_args.num_repeat,
self._global_params))
# The first block needs to take care of stride and filter size increase.
conv_block = self._get_conv_block(block_args.conv_type)
if not block_args.super_pixel: # no super_pixel at all
self._blocks.append(conv_block(block_args,
self._global_params))
else:
# if superpixel, adjust filters, kernels, and strides.
depth_factor = int(4 / block_args.strides[0] /
block_args.strides[1])
block_args = block_args._replace(
input_filters=block_args.input_filters * depth_factor,
output_filters=block_args.output_filters * depth_factor,
kernel_size=((block_args.kernel_size + 1) // 2 if
depth_factor > 1 else block_args.kernel_size))
# if the first block has stride-2 and super_pixel trandformation
if (block_args.strides[0] == 2 and block_args.strides[1] == 2):
block_args = block_args._replace(strides=[1, 1])
self._blocks.append(
conv_block(block_args, self._global_params))
block_args = block_args._replace( # sp stops at stride-2
super_pixel=0,
input_filters=input_filters,
output_filters=output_filters,
kernel_size=kernel_size)
elif block_args.super_pixel == 1:
self._blocks.append(
conv_block(block_args, self._global_params))
block_args = block_args._replace(super_pixel=2)
else:
self._blocks.append(
conv_block(block_args, self._global_params))
if block_args.num_repeat > 1: # rest of blocks with the same block_arg
# pylint: disable=protected-access
block_args = block_args._replace(
input_filters=block_args.output_filters, strides=[1, 1])
# pylint: enable=protected-access
for _ in xrange(block_args.num_repeat - 1):
self._blocks.append(conv_block(block_args,
self._global_params))
# Head part.
self._conv_head = tf.layers.Conv2D(
filters=round_filters(1280, self._global_params),
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding='same',
use_bias=False)
self._bn1 = self._batch_norm(axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon)
self._avg_pooling = tf.keras.layers.GlobalAveragePooling2D(
data_format=self._global_params.data_format)
if self._global_params.num_classes:
self._fc = tf.layers.Dense(
self._global_params.num_classes,
kernel_initializer=dense_kernel_initializer)
else:
self._fc = None
if self._global_params.dropout_rate > 0:
self._dropout = tf.keras.layers.Dropout(
self._global_params.dropout_rate)
else:
self._dropout = None
def pad_pair(self, encoder_inputs_ori, decoder_inputs_ori, seg_num,
seg_len, bucket_id, BOS):
"""
To pad the encoder_inputs and decoder_inputs to fit certain buckets in
encode-decode process.
Args:
encoder_inputs_ori: A list of length batch_size. each element is an sentence
in which each word is represented by its own id.
decoder_inputs_ori: A list of length batch_size. each element is an sentence
in which each word is represented by its own id.
seg_num: The number(No.?) of the segment that is going through the decode process.
seg_len: The length of the segment that is going through the decode process.
Equals to FLAGS.segment_length.
bucket_id: The bucket to use in the encode-decode process.
BOS: Whether this is the first segment to decode.
Returns:
batch_encoder_inputs, batch_decoder_inputs: Padded inputs that are ready
to be fed in nn.
batch_weights: Implemented as vanilla Seq2Seq.
"""
encoder_size, decoder_size = self.buckets[bucket_id]
encoder_inputs = []
decoder_inputs = []
# Get a random batch of encoder and decoder inputs from data,
# pad them if needed, reverse encoder inputs and add GO to decoder.
for i in xrange(self.batch_size):
dst_i_ori = decoder_inputs_ori[i] # the full sentence
src_i = encoder_inputs_ori[
i] + dst_i_ori[:min(len(dst_i_ori), seg_num * seg_len)]
if len(dst_i_ori) > seg_num * seg_len:
dst_i = dst_i_ori[seg_num * seg_len:(seg_num + 1) *
seg_len] # a segment
else:
dst_i = [data_utils.EOS_ID]
# Encoder inputs are padded and then reversed.
encoder_pad = [data_utils.PAD_ID] * (encoder_size - len(src_i))
encoder_inputs.append(list(reversed(src_i + encoder_pad)))
# Decoder inputs get an extra "GO" symbol, and are padded then. GO可能和BOS是一样的？
decoder_pad_size = decoder_size - len(dst_i) - 1
if BOS:
decoder_inputs.append([data_utils.GO_ID] + dst_i +
[data_utils.PAD_ID] * decoder_pad_size)
else:
decoder_inputs.append(dst_i + [data_utils.PAD_ID] *
(decoder_pad_size + 1))
# Now we create batch-major vectors from the data selected above.
batch_encoder_inputs, batch_decoder_inputs, batch_weights = [], [], []
# Batch encoder inputs are just re-indexed encoder_inputs. 有点像转置了？
for length_idx in xrange(encoder_size):
batch_encoder_inputs.append(
np.array([
encoder_inputs[batch_idx][length_idx]
for batch_idx in xrange(self.batch_size)
],
dtype=np.int32))
# Batch decoder inputs are re-indexed decoder_inputs, we create weights.
for length_idx in xrange(decoder_size):
batch_decoder_inputs.append(
np.array([
decoder_inputs[batch_idx][length_idx]
for batch_idx in xrange(self.batch_size)
],
dtype=np.int32))
# Create target_weights to be 0 for targets that are padding.
batch_weight = np.ones(self.batch_size, dtype=np.float32)
for batch_idx in xrange(self.batch_size):
# We set weight to 0 if the corresponding target is a PAD symbol.
# The corresponding target is decoder_input shifted by 1 forward.
if length_idx < decoder_size - 1:
target = decoder_inputs[batch_idx][length_idx + 1]
if length_idx == decoder_size - 1 or target == data_utils.PAD_ID:
batch_weight[batch_idx] = 0.0
batch_weights.append(batch_weight)
return batch_encoder_inputs, batch_decoder_inputs, batch_weights
def run_test():
test_list_file = 'test.list'
num_test_videos = len(list(open(test_list_file,'r')))
print("Number of test videos={}".format(num_test_videos))
# Get the sets of images and labels for training, validation, and
images_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size * gpu_num)
logits = []
for gpu_index in range(0, gpu_num):
with tf.device('/gpu:%d' % gpu_index):
logit = resnet.inference(
images_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size,:,:,:,:],
5,
False
)
logits.append(logit)
logits = tf.concat(logits,0)
norm_score = tf.nn.softmax(logits)
saver = tf.train.Saver(tf.global_variables())
config = tf.ConfigProto(allow_soft_placement = True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Create a saver for writing training checkpoints.
#saver.restore(sess, model_name)
saver.restore(sess,"./models/r3d_model-4200")
# And then after everything is built, start the training loop.
bufsize = 0
write_file = open("predict_ret.txt", "w+")
next_start_pos = 0
all_steps = int((num_test_videos - 1) / (FLAGS.batch_size * gpu_num) + 1)
accuracy = tower_acc(logits, labels_placeholder)
acc_cnt,acc5_cnt,cnt = 0,0,1
for step in xrange(all_steps):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
start_time = time.time()
test_images, test_labels, next_start_pos, _, valid_len = \
input_data.read_clip_and_label(
test_list_file,
FLAGS.batch_size * gpu_num,
start_pos=next_start_pos,
shuffle=True
)
predict_score = norm_score.eval(
session=sess,
feed_dict={images_placeholder: test_images}
)
acc5 = tf.nn.in_top_k(predict_score,test_labels,5)
top5_score = acc5.eval(session=sess,
feed_dict={images_placeholder: test_images}
)
'''
acc = sess.run(accuracy,feed_dict={
images_placeholder: test_images,
labels_placeholder: test_labels
})
print(acc)
'''
for i in range(0, valid_len):
true_label = test_labels[i],
top1_predicted_label = np.argmax(predict_score[i])
# Write results: true label, class prob for true label, predicted label, class prob for predicted label
write_file.write('{}, {}, {}, {}\n'.format(
true_label[0],
predict_score[i][true_label],
top1_predicted_label,
predict_score[i][top1_predicted_label]))
cnt += 1
if top1_predicted_label == true_label[0]:
acc_cnt += 1
if top5_score[i]:
acc5_cnt += 1
print("Test Accuracy={}".format(float(acc_cnt)/float(cnt)))
print("Top-5 Accuracy={}".format(float(acc5_cnt)/float(cnt)))
write_file.close()
print("done")
def _ps_replicas(all_workers=False):
if all_workers:
return list(range(FLAGS.ps_replicas))
# Worker K will be using replicas {0,...n-1} + K*n if we have n replicas.
num_replicas = FLAGS.ps_replicas // FLAGS.worker_replicas
return [d + FLAGS.worker_id * num_replicas for d in xrange(num_replicas)]
def __init__(self,
source_vocab_size,
target_vocab_size,
buckets,
size,
num_layers,
max_gradient_norm,
batch_size,
learning_rate,
learning_rate_decay_factor,
use_lstm=True,
num_samples=512,
forward_only=False,
dtype=tf.float32):
"""Create the model.
Args:
source_vocab_size: size of the source vocabulary.
target_vocab_size: size of the target vocabulary.
buckets: a list of pairs (I, O), where I specifies maximum input length
that will be processed in that bucket, and O specifies maximum output
length. Training instances that have inputs longer than I or outputs
longer than O will be pushed to the next bucket and padded accordingly.
We assume that the list is sorted, e.g., [(2, 4), (8, 16)].
size: number of units in each layer of the model.
num_layers: number of layers in the model.
max_gradient_norm: gradients will be clipped to maximally this norm.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
learning_rate_decay_factor: decay learning rate by this much when needed.
use_lstm: if true, we use LSTM cells instead of GRU cells.
num_samples: number of samples for sampled softmax.
forward_only: if set, we do not construct the backward pass in the model.
dtype: the data type to use to store internal variables.
"""
self.source_vocab_size = source_vocab_size
self.target_vocab_size = target_vocab_size
self.buckets = buckets
self.batch_size = batch_size
self.learning_rate = tf.Variable(float(learning_rate),
trainable=False,
dtype=dtype)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
# If we use sampled softmax, we need an output projection.
output_projection = None
softmax_loss_function = None
# Sampled softmax only makes sense if we sample less than vocabulary size.
if num_samples > 0 and num_samples < self.target_vocab_size:
w_t = tf.get_variable("proj_w", [self.target_vocab_size, size],
dtype=dtype)
w = tf.transpose(w_t)
b = tf.get_variable("proj_b", [self.target_vocab_size],
dtype=dtype)
output_projection = (w, b)
def sampled_loss(inputs, labels):
labels = tf.reshape(labels, [-1, 1])
# We need to compute the sampled_softmax_loss using 32bit floats to
# avoid numerical instabilities.
local_w_t = tf.cast(w_t, tf.float32)
local_b = tf.cast(b, tf.float32)
local_inputs = tf.cast(inputs, tf.float32)
return tf.cast(
tf.nn.sampled_softmax_loss(local_w_t, local_b,
local_inputs, labels,
num_samples,
self.target_vocab_size), dtype)
softmax_loss_function = sampled_loss
# Create the internal multi-layer cell for our RNN.
list_of_cell = []
for layer in xrange(num_layers):
if layer % 2 == 0:
with tf.device('/gpu:0'):
single_cell = tf.nn.rnn_cell.LSTMCell(size)
list_of_cell.append(single_cell)
else:
with tf.device('/gpu:1'):
single_cell = tf.nn.rnn_cell.LSTMCell(size)
list_of_cell.append(single_cell)
if num_layers > 1:
cell = Stack_Residual_RNNCell.Stack_Residual_RNNCell(list_of_cell)
# The seq2seq function: we use embedding for the input and attention.
def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
return seq2seq_for_MT.embedding_attention_seq2seq(
encoder_inputs,
decoder_inputs,
cell,
num_layers=num_layers,
num_encoder_symbols=source_vocab_size,
num_decoder_symbols=target_vocab_size,
embedding_size=512,
output_projection=output_projection,
feed_previous=do_decode,
dtype=dtype)
# Feeds for inputs.
self.encoder_inputs = []
self.decoder_inputs = []
self.target_weights = []
for i in xrange(buckets[-1][0]): # Last bucket is the biggest one.
self.encoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="encoder{0}".format(i)))
for i in xrange(buckets[-1][1] + 1):
self.decoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="decoder{0}".format(i)))
self.target_weights.append(
tf.placeholder(dtype, shape=[None],
name="weight{0}".format(i)))
# Our targets are decoder inputs shifted by one. 感觉这里有问题
targets = [
self.decoder_inputs[i + 1]
for i in xrange(len(self.decoder_inputs) - 1)
]
# Training outputs and losses.
if forward_only:
self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
self.encoder_inputs,
self.decoder_inputs,
targets,
self.target_weights,
buckets,
lambda x, y: seq2seq_f(x, y, True),
softmax_loss_function=softmax_loss_function)
# If we use output projection, we need to project outputs for decoding.
if output_projection is not None:
for b in xrange(len(buckets)):
self.outputs[b] = [
tf.matmul(output, output_projection[0]) +
output_projection[1] for output in self.outputs[b]
]
else:
self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
self.encoder_inputs,
self.decoder_inputs,
targets,
self.target_weights,
buckets,
lambda x, y: seq2seq_f(x, y, False),
softmax_loss_function=softmax_loss_function)
# Gradients and SGD update operation for training the model.
params = tf.trainable_variables()
if not forward_only:
self.gradient_norms = []
self.updates = []
opt = tf.train.GradientDescentOptimizer(self.learning_rate)
for b in xrange(len(buckets)):
gradients = tf.gradients(self.losses[b], params)
clipped_gradients, norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.gradient_norms.append(norm)
self.updates.append(
opt.apply_gradients(zip(clipped_gradients, params),
global_step=self.global_step))
self.saver = tf.train.Saver(tf.all_variables())
def _batches(iterable, size):
for i in xrange(0, len(iterable), size):
yield iterable[i:i + size]
def step(self, session, encoder_inputs, decoder_inputs, target_weights,
bucket_id, forward_only):
"""Run a step of the model feeding the given inputs.
Args:
session: tensorflow session to use.
encoder_inputs: list of numpy int vectors to feed as encoder inputs.
decoder_inputs: list of numpy int vectors to feed as decoder inputs.
target_weights: list of numpy float vectors to feed as target weights.
bucket_id: which bucket of the model to use.
forward_only: whether to do the backward step or only forward.
Returns:
A triple consisting of gradient norm (or None if we did not do backward),
average perplexity, and the outputs.
Raises:
ValueError: if length of encoder_inputs, decoder_inputs, or
target_weights disagrees with bucket size for the specified bucket_id.
"""
# Check if the sizes match.
encoder_size, decoder_size = self.buckets[bucket_id]
if len(encoder_inputs) != encoder_size:
raise ValueError(
"Encoder length must be equal to the one in bucket,"
" %d != %d." % (len(encoder_inputs), encoder_size))
if len(decoder_inputs) != decoder_size:
raise ValueError(
"Decoder length must be equal to the one in bucket,"
" %d != %d." % (len(decoder_inputs), decoder_size))
if len(target_weights) != decoder_size:
raise ValueError(
"Weights length must be equal to the one in bucket,"
" %d != %d." % (len(target_weights), decoder_size))
# Input feed: encoder inputs, decoder inputs, target_weights, as provided.
input_feed = {}
for l in xrange(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
for l in xrange(decoder_size):
input_feed[self.decoder_inputs[l].name] = decoder_inputs[l]
input_feed[self.target_weights[l].name] = target_weights[l]
# Since our targets are decoder inputs shifted by one, we need one more.
last_target = self.decoder_inputs[decoder_size].name
input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32)
# Output feed: depends on whether we do a backward step or not.
if not forward_only:
output_feed = [
self.updates[bucket_id], # Update Op that does SGD.
self.gradient_norms[bucket_id], # Gradient norm.
self.losses[bucket_id]
] # Loss for this batch.
else:
output_feed = [self.losses[bucket_id]] # Loss for this batch.
for l in xrange(decoder_size): # Output logits.
output_feed.append(self.outputs[bucket_id][l])
outputs = session.run(output_feed, input_feed)
if not forward_only:
return outputs[1], outputs[
2], None # Gradient norm, loss, no outputs.
else:
return None, outputs[0], outputs[
1:] # No gradient norm, loss, outputs.