def model(inputs, is_training):
"""Creation of the model graph."""
tf.logging.info(FLAGS.model_structure)
tf.logging.info(FLAGS.model_edge_weights)
structure = json.loads(FLAGS.model_structure)
feature_shape = inputs.shape
batch_size = feature_shape[0] // FLAGS.num_frames
original_num_frames = FLAGS.num_frames
grouping = {-3: [], -2: [], -1: [], 0: [], 1: [], 2: [], 3: []}
for i in range(len(structure)):
grouping[structure[i][0]].append(i)
stem_count = len(grouping[-3]) + len(grouping[-2]) + len(grouping[-1])
assert stem_count != 0
stem_filters = 128 // stem_count
original_inputs = inputs
if grouping[-2]:
# Instead of loading optical flows as inputs from data pipeline, we are
# applying the "Representation Flow" to RGB frames so that we can compute
# the flow within TPU/GPU on fly. It's essentially optical flow since we
# do it with RGBs.
flow_inputs = rf.rep_flow(original_inputs,
batch_size,
original_num_frames,
num_iter=40,
is_training=is_training,
bottleneck=1,
scope='rep_flow')
streams = []
for i in range(len(structure)):
with tf.variable_scope('Node_' + str(i)):
if structure[i][0] == -1:
inputs = rgb_conv_stem(original_inputs,
original_num_frames, stem_filters,
structure[i][1], is_training,
data_format)
streams.append(inputs)
elif structure[i][0] == -2:
inputs = flow_conv_stem(flow_inputs, stem_filters,
structure[i][1], is_training,
data_format)
streams.append(inputs)
else:
num_frames = original_num_frames
block_number = structure[i][0]
combined_inputs = []
if combine_method == 'concat':
combined_inputs = [
streams[structure[i][1][j]]
for j in range(0, len(structure[i][1]))
]
combined_inputs = spatial_resize_and_concat(
combined_inputs)
else:
combined_inputs = [
streams[structure[i][1][j]]
for j in range(0, len(structure[i][1]))
]
combined_inputs = multi_connection_fusion(
combined_inputs, index=i)
graph = block_group(inputs=combined_inputs,
filters=structure[i][2],
block_fn=block_fn,
blocks=layers[block_number],
strides=structure[i][4],
is_training=is_training,
name='block_group' + str(i),
block_level=structure[i][0],
num_frames=num_frames,
temporal_dilation=structure[i][3],
data_format=data_format)
streams.append(graph)
outputs = multi_stream_heads(streams, grouping[3], original_num_frames,
num_classes, data_format)
return outputs
def model(inputs, is_training):
"""Creation of the model graph."""
tf.logging.info(FLAGS.model_structure)
tf.logging.info(FLAGS.model_edge_weights)
structure = json.loads(FLAGS.model_structure)
if FLAGS.use_object_input:
feature_shape = inputs[0].shape
original_inputs = inputs[0]
object_inputs = inputs[1]
else:
feature_shape = inputs.shape
original_inputs = inputs
object_inputs = None
batch_size = feature_shape[0] // FLAGS.num_frames
original_num_frames = FLAGS.num_frames
num_frames = original_num_frames
grouping = {-3: [], -2: [], -1: [], 0: [], 1: [], 2: [], 3: []}
for i in range(len(structure)):
grouping[structure[i][0]].append(i)
stem_count = len(grouping[-3]) + len(grouping[-2]) + len(grouping[-1])
assert stem_count != 0
stem_filters = 128 // stem_count
if grouping[-2]:
# Instead of loading optical flows as inputs from data pipeline, we are
# applying the "Representation Flow" to RGB frames so that we can compute
# the flow within TPU/GPU on fly. It's essentially optical flow since we
# do it with RGBs.
flow_inputs = rf.rep_flow(original_inputs,
batch_size,
original_num_frames,
num_iter=40,
is_training=is_training,
bottleneck=1,
scope='rep_flow')
streams = []
for i in range(len(structure)):
with tf.variable_scope('Node_' + str(i)):
if structure[i][0] == -1:
inputs = asn.rgb_conv_stem(original_inputs,
original_num_frames,
stem_filters, structure[i][1],
is_training, data_format)
streams.append(inputs)
elif structure[i][0] == -2:
inputs = asn.flow_conv_stem(flow_inputs, stem_filters,
structure[i][1], is_training,
data_format)
streams.append(inputs)
elif structure[i][0] == -3:
# In order to use the object inputs, you need to feed your object
# input tensor here.
inputs = object_conv_stem(object_inputs, data_format)
streams.append(inputs)
else:
block_number = structure[i][0]
combined_inputs = [
streams[structure[i][1][j]]
for j in range(0, len(structure[i][1]))
]
tf.logging.info(grouping)
nodes_below = []
for k in range(-3, structure[i][0]):
nodes_below = nodes_below + grouping[k]
peers = []
if FLAGS.attention_mode:
lg_channel = -1
tf.logging.info(nodes_below)
for k in nodes_below:
tf.logging.info(streams[k].shape)
lg_channel = max(streams[k].shape[3], lg_channel)
for node_index in nodes_below:
attn = tf.reduce_mean(streams[node_index], [1, 2])
attn = tf.layers.dense(
inputs=attn,
units=lg_channel,
kernel_initializer=tf.
random_normal_initializer(stddev=.01))
peers.append(attn)
combined_inputs = fusion_with_peer_attention(
combined_inputs,
index=i,
attention_mode=FLAGS.attention_mode,
attention_in=peers,
use_5d_mode=False,
data_format=data_format)
graph = asn.block_group(inputs=combined_inputs,
filters=structure[i][2],
block_fn=block_fn,
blocks=layers[block_number],
strides=structure[i][4],
is_training=is_training,
name='block_group' + str(i),
block_level=structure[i][0],
num_frames=num_frames,
temporal_dilation=structure[i][3],
data_format=data_format)
streams.append(graph)
outputs = asn.multi_stream_heads(streams, grouping[3],
original_num_frames, num_classes,
data_format)
return outputs