def make_perf_test_graph(self, width, height, scale=1.0, placeholders=[]):
data = nd.Struct()
data.width = width
data.height = height
data.img = nd.Struct()
data.img.L = nd.ops.ones(1, 3, height, width)
data.img.R = nd.ops.ones(1, 3, height, width)
output = self.translate_output(StandardDeployment().make_graph(
data=data,
net_graph_constructor=lambda data: self.make_net_graph(
data, include_losses=False),
divisor=self._deploy_divisor,
scale=scale))
return output
def make_eval_graph(self, width, height, scale=1.0):
data = nd.Struct()
data.width = width
data.height = height
data.img = nd.Struct()
data.img[0] = nd.placeholder('data.img0', (1, 3, height, width))
data.img[1] = nd.placeholder('data.img1', (1, 3, height, width))
output = self.translate_output(StandardDeployment().make_graph(
data=data,
net_graph_constructor=lambda data: self.make_net_graph(
data, include_losses=False),
divisor=self._deploy_divisor))
return output
def make_graph(self, input):
out = nd.Struct()
out.make_struct('levels')
with nd.Scope('encoder'):
conv0 = nd.scope.conv_nl(input, name="conv0", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv0'])
conv1 = nd.scope.conv_nl(conv0, name="conv1", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv1'])
conv1_1 = nd.scope.conv_nl(conv1, name="conv1_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv1_1'])
conv2 = nd.scope.conv_nl(conv1_1, name="conv2", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv2'])
conv2_1 = nd.scope.conv_nl(conv2, name="conv2_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv2_1'])
prediction2 = self.predict(conv2_1, level=2, loss_weight=self._loss_weights['level2'], out=out)
with nd.Scope('decoder'):
decoder1, prediction1 = \
self.refine(level=1,
input=conv2_1,
input_prediction=prediction2,
features=conv1_1, out=out)
if self._exit_after == 1:
out.final = out.levels[1]
return out
decoder0, prediction0 = \
self.refine(level=0,
input=decoder1,
input_prediction=prediction1,
features=conv0, out=out)
out.final = out.levels[0]
return out
def make_graph(self, data, net_graph_constructor, divisor=64., scale=1.0):
self._scale = scale
self._width = data.width
self._height = data.height
self._temp_width = ceil(self._width * self._scale / divisor) * divisor
self._temp_height = ceil(
self._height * self._scale / divisor) * divisor
self._rescale_coeff_x = self._width / self._temp_width
self._rescale_coeff_y = self._height / self._temp_height
nd.log('data:')
nd.log(data)
input = nd.Struct()
data.map('img', self.input_image_resample, input)
data.map('flow', self.input_resample_flow, input)
data.map('disp', self.input_resample_disp, input)
data.map('occ', self.input_resample_binary, input)
data.map('mb', self.input_resample_binary, input)
data.map('db', self.input_resample_binary, input)
pred = net_graph_constructor(input)
nd.log('pred:')
nd.log(pred)
if isinstance(pred, (tuple)):
pred = pred[0]
if isinstance(pred, (list)):
output = []
for prediction in pred:
out = nd.Struct()
self.map_output(prediction.final, out)
output.append(out)
else:
output = nd.Struct()
self.map_output(pred.final, output)
nd.log('output:')
nd.log(output)
return output
def translate_output(self, data):
data2 = nd.Struct()
data.copy(data2)
data2.translate('db', 'db_soft', softmax2_soft_translator)
data2.translate('db', 'db', softmax2_hard_translator)
data2.translate('occ', 'occ_soft', softmax2_soft_translator)
data2.translate('occ', 'occ', softmax2_hard_translator)
return data2
def concat(self, other):
dest = nd.Struct()
for member in list(self._members.keys()):
if self.is_data(member):
dest[member] = nd.ops.concat(self[member], other[member])
elif self.is_struct(member):
dest[member] = self[member].concat(other[member])
else:
dest[member] = self[member]
return dest
def copy(self, other, mapper=None):
if mapper is None: mapper = lambda x: x
for member in list(self._members.keys()):
if self.is_data(member):
other[member] = mapper(self[member])
elif self.is_struct(member):
other[member] = nd.Struct()
self[member].copy(other[member], mapper)
else:
other[member] = self[member]
def translate_output(self, data):
data2 = nd.Struct()
data.copy(data2)
data2.translate('mb', 'mb_soft', softmax2_soft_translator)
data2.translate('mb', 'mb', softmax2_hard_translator)
data2.translate('occ', 'occ_soft', softmax2_soft_translator)
data2.translate('occ', 'occ', softmax2_hard_translator)
data2.translate('iul_b_log', 'iul_scale', iul_b_log_translator)
data2.translate('iul_scale', 'iul_entropy', iul_b_log_ent_translator)
return data2
def make_graph(self, feat0, feat1):
data = nd.Struct()
(data.conv1a, data.conv1b) = nd.scope.conv_nl((feat0, feat1), name='conv1_map', kernel_size=self._kernel_sizes[0], stride=self._strides[0], pad=int(self._kernel_sizes[0]/2), num_output=int(self._feature_channels[0]))
if self._exit_after == 1: return data
(data.conv2a, data.conv2b) = nd.scope.conv_nl((data.conv1a, data.conv1b), name='conv2_map', kernel_size=self._kernel_sizes[1], stride=self._strides[1], pad=int(self._kernel_sizes[1]/2), num_output=int(self._feature_channels[1]))
if self._exit_after == 2: return data
(data.conv3a, data.conv3b) = nd.scope.conv_nl((data.conv2a, data.conv2b), name='conv3_map', kernel_size=self._kernel_sizes[2], stride=self._strides[2], pad=int(self._kernel_sizes[2]/2), num_output=int(self._feature_channels[2]))
if self._exit_after == 3: return data
return data
def make_graph(self, img0, img1):
data = nd.Struct()
(data.conv1a, data.conv1b) = nd.scope.conv_nl((img0, img1), name='conv1', kernel_size=7, stride=2, pad=3, num_output=self._feature_channels['conv1'])
if self._exit_after == 1: return data
(data.conv2a, data.conv2b) = nd.scope.conv_nl((data.conv1a, data.conv1b), name='conv2', kernel_size=5, stride=2, pad=2, num_output=self._feature_channels['conv2'])
if self._exit_after == 2: return data
(data.conv3a, data.conv3b) = nd.scope.conv_nl((data.conv2a, data.conv2b), name='conv3', kernel_size=5, stride=2, pad=2, num_output=self._feature_channels['conv3'])
if self._exit_after == 3: return data
return data
def disassemble(self, prediction, **kwargs):
config = self.slice_config()
if len(config) == 0: slices = [
prediction,
]
else: slices = nd.ops.slice(prediction, self.slice_config())
i = 0
def get_slice(scale):
nonlocal i
s = slices[i]
i += 1
if scale != 1: s = nd.ops.scale(s, scale)
if id.mod_func is not None: s = id.mod_func(s, **kwargs)
return s
data = nd.Struct()
for id in self._ids:
if id.dist is not None and id.dist > 1 and not self._multiframe:
raise BaseException(
'found a dist > 1 in data, but multiframe is False')
if self._multiframe:
raise NotImplementedError
s = data
n = None
p = None # parent
def descend(name):
nonlocal s, n, p
if name is not None:
s.make_struct(name)
p = s
n = name
s = s[n]
descend(id.type)
descend(id.perspective)
descend(id.offset)
descend(id.dist)
descend(id.dir)
if id.array:
for j in range(0, id.array_length):
p[n][j] = get_slice(id.scale)
else:
p[n] = get_slice(id.scale)
return data
def translate(self, old_name, new_name, mapper, remove=False):
def map(x):
y = x
if nd.is_list(mapper):
for m in mapper:
y = m(y)
else:
y = mapper(y)
return y
for member in list(self._members.keys()):
if self.is_data(member):
if member == old_name:
self[new_name] = map(self[old_name])
if remove: del self[old_name]
elif self.is_struct(member):
if member == old_name:
ns = nd.Struct()
self[old_name].copy(ns, mapper)
if remove: del self[old_name]
self[new_name] = ns
else:
self[member].translate(old_name, new_name, mapper, remove)
def make_struct(self, member):
if not member in self._members: self[member] = nd.Struct()
def make_graph(self, input, edge_features=None):
out = nd.Struct()
out.make_struct('levels')
with nd.Scope('encoder'):
conv1 = nd.scope.conv_nl(
input,
name="conv1",
kernel_size=7,
stride=2,
pad=3,
num_output=self._encoder_channels['conv1'])
conv2 = nd.scope.conv_nl(
conv1,
name="conv2",
kernel_size=5,
stride=2,
pad=2,
num_output=self._encoder_channels['conv2'])
conv3 = nd.scope.conv_nl(
conv2,
name="conv3",
kernel_size=5,
stride=2,
pad=2,
num_output=self._encoder_channels['conv3'])
conv3_1 = nd.scope.conv_nl(
conv3,
name="conv3_1",
kernel_size=3,
stride=1,
pad=1,
num_output=self._encoder_channels['conv3_1'])
conv4 = nd.scope.conv_nl(
conv3_1,
name="conv4",
kernel_size=3,
stride=2,
pad=1,
num_output=self._encoder_channels['conv4'])
conv4_1 = nd.scope.conv_nl(
conv4,
name="conv4_1",
kernel_size=3,
stride=1,
pad=1,
num_output=self._encoder_channels['conv4_1'])
if self._encoder_level == 4:
prediction4 = self.predict(
conv4_1,
level=4,
loss_weight=self._loss_weights['level4'],
out=out)
else:
conv5 = nd.scope.conv_nl(
conv4_1,
name="conv5",
kernel_size=3,
stride=2,
pad=1,
num_output=self._encoder_channels['conv5'])
conv5_1 = nd.scope.conv_nl(
conv5,
name="conv5_1",
kernel_size=3,
stride=1,
pad=1,
num_output=self._encoder_channels['conv5_1'])
if self._encoder_level == 5:
prediction5 = self.predict(
conv5_1,
level=5,
loss_weight=self._loss_weights['level5'],
out=out)
else:
conv6 = nd.scope.conv_nl(
conv5_1,
name="conv6",
kernel_size=3,
stride=2,
pad=1,
num_output=self._encoder_channels['conv6'])
conv6_1 = nd.scope.conv_nl(
conv6,
name="conv6_1",
kernel_size=3,
stride=1,
pad=1,
num_output=self._encoder_channels['conv6_1'])
prediction6 = self.predict(
conv6_1,
level=6,
loss_weight=self._loss_weights['level6'],
out=out)
with nd.Scope('decoder'):
if self._encoder_level >= 6:
decoder5, prediction5 = \
self.refine(level=5,
input=conv6_1,
input_prediction=prediction6,
features=conv5_1, out=out)
if self._exit_after == 5:
out.final = out.levels[5]
return out
if self._encoder_level >= 5:
decoder4, prediction4 = \
self.refine(level=4,
input=decoder5 if self._encoder_level > 5 else conv5_1,
input_prediction=prediction5,
features=conv4_1, out=out)
if self._exit_after == 4:
out.final = out.levels[4]
return out
decoder3, prediction3 = \
self.refine(level=3,
input=decoder4 if self._encoder_level > 4 else conv4_1,
input_prediction=prediction4,
features=conv3_1, out=out)
if self._exit_after == 3:
out.final = out.levels[3]
return out
decoder2, prediction2 = \
self.refine(level=2,
input=decoder3,
input_prediction=prediction3,
features=conv2, out=out)
if self._exit_after == 2:
out.final = out.levels[2]
return out
decoder1, prediction1 = \
self.refine(level=1,
input=decoder2,
input_prediction=prediction2,
features=conv1, out=out)
if self._exit_after == 1:
out.final = out.levels[1]
return out
if edge_features is None:
raise BaseException(
'Architecture_S needs edge features if now shallow')
edges = nd.scope.conv_nl(
edge_features,
name="conv_edges",
kernel_size=3,
stride=1,
pad=1,
num_output=self._decoder_channels['level0'])
decoder0, prediction0 = \
self.refine(level=0,
input=decoder1,
input_prediction=prediction1,
features=edges, out=out)
out.final = out.levels[0]
return out
def make_graph(self, input, edge_features=None, use_1D_corr=False, single_direction=0):
out = nd.Struct()
out.make_struct('levels')
if use_1D_corr:
corr = nd.ops.correlation_1d(input.conv3a, input.conv3b,
kernel_size=1,
max_displacement=40,
pad=40,
stride_1=1,
stride_2=1,
single_direction=single_direction
)
else:
corr = nd.ops.correlation_2d(input.conv3a, input.conv3b,
kernel_size=1,
max_displacement=20,
pad=20,
stride_1=1,
stride_2=2)
with nd.Scope('encoder'):
redir = nd.scope.conv_nl(input.conv3a, name="conv_redir", kernel_size=1, stride=1, pad=0, num_output=self._encoder_channels['conv_redir'])
merged = nd.ops.concat(redir, corr)
conv3_1 = nd.scope.conv_nl(merged, name="conv3_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv3_1'])
conv4 = nd.scope.conv_nl(conv3_1, name="conv4", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv4'])
conv4_1 = nd.scope.conv_nl(conv4, name="conv4_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv4_1'])
conv5 = nd.scope.conv_nl(conv4_1, name="conv5", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv5'])
conv5_1 = nd.scope.conv_nl(conv5, name="conv5_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv5_1'])
conv6 = nd.scope.conv_nl(conv5_1, name="conv6", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv6'])
conv6_1 = nd.scope.conv_nl(conv6, name="conv6_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv6_1'])
prediction6 = self.predict(conv6_1, level=6, loss_weight=self._loss_weights['level6'], out=out)
with nd.Scope('decoder'):
decoder5, prediction5 = \
self.refine(level=5,
input=conv6_1,
input_prediction=prediction6,
features=conv5_1, out=out)
if self._exit_after == 5:
out.final = out.levels[5]
return out
decoder4, prediction4 = \
self.refine(level=4,
input=decoder5,
input_prediction=prediction5,
features=conv4_1, out=out)
if self._exit_after == 4:
out.final = out.levels[4]
return out
decoder3, prediction3 = \
self.refine(level=3,
input=decoder4,
input_prediction=prediction4,
features=conv3_1, out=out)
if self._exit_after == 3:
out.final = out.levels[3]
return out
decoder2, prediction2 = \
self.refine(level=2,
input=decoder3,
input_prediction=prediction3,
features=input.conv2a, out=out)
if self._exit_after == 2:
out.final = out.levels[2]
return out
decoder1, prediction1 = \
self.refine(level=1,
input=decoder2,
input_prediction=prediction2,
features=input.conv1a, out=out)
if self._exit_after == 1:
out.final = out.levels[1]
return out
if edge_features is None:
raise BaseException('Architecture_S needs edge features if now shallow')
edges = nd.scope.conv_nl(edge_features,
name="conv_edges",
kernel_size=3,
stride=1,
pad=1,
num_output=self._decoder_channels['level0'])
decoder0, prediction0 = \
self.refine(level=0,
input=decoder1,
input_prediction=prediction1,
features=edges, out=out)
out.final = out.levels[0]
return out
