def make_perf_test_graph(self, width, height, scale=1.0, placeholders=[]): data = nd.Struct() data.width = width data.height = height data.img = nd.Struct() data.img.L = nd.ops.ones(1, 3, height, width) data.img.R = nd.ops.ones(1, 3, height, width) output = self.translate_output(StandardDeployment().make_graph( data=data, net_graph_constructor=lambda data: self.make_net_graph( data, include_losses=False), divisor=self._deploy_divisor, scale=scale)) return output
def make_eval_graph(self, width, height, scale=1.0): data = nd.Struct() data.width = width data.height = height data.img = nd.Struct() data.img[0] = nd.placeholder('data.img0', (1, 3, height, width)) data.img[1] = nd.placeholder('data.img1', (1, 3, height, width)) output = self.translate_output(StandardDeployment().make_graph( data=data, net_graph_constructor=lambda data: self.make_net_graph( data, include_losses=False), divisor=self._deploy_divisor)) return output
def make_graph(self, input): out = nd.Struct() out.make_struct('levels') with nd.Scope('encoder'): conv0 = nd.scope.conv_nl(input, name="conv0", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv0']) conv1 = nd.scope.conv_nl(conv0, name="conv1", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv1']) conv1_1 = nd.scope.conv_nl(conv1, name="conv1_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv1_1']) conv2 = nd.scope.conv_nl(conv1_1, name="conv2", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv2']) conv2_1 = nd.scope.conv_nl(conv2, name="conv2_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv2_1']) prediction2 = self.predict(conv2_1, level=2, loss_weight=self._loss_weights['level2'], out=out) with nd.Scope('decoder'): decoder1, prediction1 = \ self.refine(level=1, input=conv2_1, input_prediction=prediction2, features=conv1_1, out=out) if self._exit_after == 1: out.final = out.levels[1] return out decoder0, prediction0 = \ self.refine(level=0, input=decoder1, input_prediction=prediction1, features=conv0, out=out) out.final = out.levels[0] return out
def make_graph(self, data, net_graph_constructor, divisor=64., scale=1.0): self._scale = scale self._width = data.width self._height = data.height self._temp_width = ceil(self._width * self._scale / divisor) * divisor self._temp_height = ceil( self._height * self._scale / divisor) * divisor self._rescale_coeff_x = self._width / self._temp_width self._rescale_coeff_y = self._height / self._temp_height nd.log('data:') nd.log(data) input = nd.Struct() data.map('img', self.input_image_resample, input) data.map('flow', self.input_resample_flow, input) data.map('disp', self.input_resample_disp, input) data.map('occ', self.input_resample_binary, input) data.map('mb', self.input_resample_binary, input) data.map('db', self.input_resample_binary, input) pred = net_graph_constructor(input) nd.log('pred:') nd.log(pred) if isinstance(pred, (tuple)): pred = pred[0] if isinstance(pred, (list)): output = [] for prediction in pred: out = nd.Struct() self.map_output(prediction.final, out) output.append(out) else: output = nd.Struct() self.map_output(pred.final, output) nd.log('output:') nd.log(output) return output
def translate_output(self, data): data2 = nd.Struct() data.copy(data2) data2.translate('db', 'db_soft', softmax2_soft_translator) data2.translate('db', 'db', softmax2_hard_translator) data2.translate('occ', 'occ_soft', softmax2_soft_translator) data2.translate('occ', 'occ', softmax2_hard_translator) return data2
def concat(self, other): dest = nd.Struct() for member in list(self._members.keys()): if self.is_data(member): dest[member] = nd.ops.concat(self[member], other[member]) elif self.is_struct(member): dest[member] = self[member].concat(other[member]) else: dest[member] = self[member] return dest
def copy(self, other, mapper=None): if mapper is None: mapper = lambda x: x for member in list(self._members.keys()): if self.is_data(member): other[member] = mapper(self[member]) elif self.is_struct(member): other[member] = nd.Struct() self[member].copy(other[member], mapper) else: other[member] = self[member]
def translate_output(self, data): data2 = nd.Struct() data.copy(data2) data2.translate('mb', 'mb_soft', softmax2_soft_translator) data2.translate('mb', 'mb', softmax2_hard_translator) data2.translate('occ', 'occ_soft', softmax2_soft_translator) data2.translate('occ', 'occ', softmax2_hard_translator) data2.translate('iul_b_log', 'iul_scale', iul_b_log_translator) data2.translate('iul_scale', 'iul_entropy', iul_b_log_ent_translator) return data2
def make_graph(self, feat0, feat1): data = nd.Struct() (data.conv1a, data.conv1b) = nd.scope.conv_nl((feat0, feat1), name='conv1_map', kernel_size=self._kernel_sizes[0], stride=self._strides[0], pad=int(self._kernel_sizes[0]/2), num_output=int(self._feature_channels[0])) if self._exit_after == 1: return data (data.conv2a, data.conv2b) = nd.scope.conv_nl((data.conv1a, data.conv1b), name='conv2_map', kernel_size=self._kernel_sizes[1], stride=self._strides[1], pad=int(self._kernel_sizes[1]/2), num_output=int(self._feature_channels[1])) if self._exit_after == 2: return data (data.conv3a, data.conv3b) = nd.scope.conv_nl((data.conv2a, data.conv2b), name='conv3_map', kernel_size=self._kernel_sizes[2], stride=self._strides[2], pad=int(self._kernel_sizes[2]/2), num_output=int(self._feature_channels[2])) if self._exit_after == 3: return data return data
def make_graph(self, img0, img1): data = nd.Struct() (data.conv1a, data.conv1b) = nd.scope.conv_nl((img0, img1), name='conv1', kernel_size=7, stride=2, pad=3, num_output=self._feature_channels['conv1']) if self._exit_after == 1: return data (data.conv2a, data.conv2b) = nd.scope.conv_nl((data.conv1a, data.conv1b), name='conv2', kernel_size=5, stride=2, pad=2, num_output=self._feature_channels['conv2']) if self._exit_after == 2: return data (data.conv3a, data.conv3b) = nd.scope.conv_nl((data.conv2a, data.conv2b), name='conv3', kernel_size=5, stride=2, pad=2, num_output=self._feature_channels['conv3']) if self._exit_after == 3: return data return data
def disassemble(self, prediction, **kwargs): config = self.slice_config() if len(config) == 0: slices = [ prediction, ] else: slices = nd.ops.slice(prediction, self.slice_config()) i = 0 def get_slice(scale): nonlocal i s = slices[i] i += 1 if scale != 1: s = nd.ops.scale(s, scale) if id.mod_func is not None: s = id.mod_func(s, **kwargs) return s data = nd.Struct() for id in self._ids: if id.dist is not None and id.dist > 1 and not self._multiframe: raise BaseException( 'found a dist > 1 in data, but multiframe is False') if self._multiframe: raise NotImplementedError s = data n = None p = None # parent def descend(name): nonlocal s, n, p if name is not None: s.make_struct(name) p = s n = name s = s[n] descend(id.type) descend(id.perspective) descend(id.offset) descend(id.dist) descend(id.dir) if id.array: for j in range(0, id.array_length): p[n][j] = get_slice(id.scale) else: p[n] = get_slice(id.scale) return data
def translate(self, old_name, new_name, mapper, remove=False): def map(x): y = x if nd.is_list(mapper): for m in mapper: y = m(y) else: y = mapper(y) return y for member in list(self._members.keys()): if self.is_data(member): if member == old_name: self[new_name] = map(self[old_name]) if remove: del self[old_name] elif self.is_struct(member): if member == old_name: ns = nd.Struct() self[old_name].copy(ns, mapper) if remove: del self[old_name] self[new_name] = ns else: self[member].translate(old_name, new_name, mapper, remove)
def make_struct(self, member): if not member in self._members: self[member] = nd.Struct()
def make_graph(self, input, edge_features=None): out = nd.Struct() out.make_struct('levels') with nd.Scope('encoder'): conv1 = nd.scope.conv_nl( input, name="conv1", kernel_size=7, stride=2, pad=3, num_output=self._encoder_channels['conv1']) conv2 = nd.scope.conv_nl( conv1, name="conv2", kernel_size=5, stride=2, pad=2, num_output=self._encoder_channels['conv2']) conv3 = nd.scope.conv_nl( conv2, name="conv3", kernel_size=5, stride=2, pad=2, num_output=self._encoder_channels['conv3']) conv3_1 = nd.scope.conv_nl( conv3, name="conv3_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv3_1']) conv4 = nd.scope.conv_nl( conv3_1, name="conv4", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv4']) conv4_1 = nd.scope.conv_nl( conv4, name="conv4_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv4_1']) if self._encoder_level == 4: prediction4 = self.predict( conv4_1, level=4, loss_weight=self._loss_weights['level4'], out=out) else: conv5 = nd.scope.conv_nl( conv4_1, name="conv5", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv5']) conv5_1 = nd.scope.conv_nl( conv5, name="conv5_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv5_1']) if self._encoder_level == 5: prediction5 = self.predict( conv5_1, level=5, loss_weight=self._loss_weights['level5'], out=out) else: conv6 = nd.scope.conv_nl( conv5_1, name="conv6", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv6']) conv6_1 = nd.scope.conv_nl( conv6, name="conv6_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv6_1']) prediction6 = self.predict( conv6_1, level=6, loss_weight=self._loss_weights['level6'], out=out) with nd.Scope('decoder'): if self._encoder_level >= 6: decoder5, prediction5 = \ self.refine(level=5, input=conv6_1, input_prediction=prediction6, features=conv5_1, out=out) if self._exit_after == 5: out.final = out.levels[5] return out if self._encoder_level >= 5: decoder4, prediction4 = \ self.refine(level=4, input=decoder5 if self._encoder_level > 5 else conv5_1, input_prediction=prediction5, features=conv4_1, out=out) if self._exit_after == 4: out.final = out.levels[4] return out decoder3, prediction3 = \ self.refine(level=3, input=decoder4 if self._encoder_level > 4 else conv4_1, input_prediction=prediction4, features=conv3_1, out=out) if self._exit_after == 3: out.final = out.levels[3] return out decoder2, prediction2 = \ self.refine(level=2, input=decoder3, input_prediction=prediction3, features=conv2, out=out) if self._exit_after == 2: out.final = out.levels[2] return out decoder1, prediction1 = \ self.refine(level=1, input=decoder2, input_prediction=prediction2, features=conv1, out=out) if self._exit_after == 1: out.final = out.levels[1] return out if edge_features is None: raise BaseException( 'Architecture_S needs edge features if now shallow') edges = nd.scope.conv_nl( edge_features, name="conv_edges", kernel_size=3, stride=1, pad=1, num_output=self._decoder_channels['level0']) decoder0, prediction0 = \ self.refine(level=0, input=decoder1, input_prediction=prediction1, features=edges, out=out) out.final = out.levels[0] return out
def make_graph(self, input, edge_features=None, use_1D_corr=False, single_direction=0): out = nd.Struct() out.make_struct('levels') if use_1D_corr: corr = nd.ops.correlation_1d(input.conv3a, input.conv3b, kernel_size=1, max_displacement=40, pad=40, stride_1=1, stride_2=1, single_direction=single_direction ) else: corr = nd.ops.correlation_2d(input.conv3a, input.conv3b, kernel_size=1, max_displacement=20, pad=20, stride_1=1, stride_2=2) with nd.Scope('encoder'): redir = nd.scope.conv_nl(input.conv3a, name="conv_redir", kernel_size=1, stride=1, pad=0, num_output=self._encoder_channels['conv_redir']) merged = nd.ops.concat(redir, corr) conv3_1 = nd.scope.conv_nl(merged, name="conv3_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv3_1']) conv4 = nd.scope.conv_nl(conv3_1, name="conv4", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv4']) conv4_1 = nd.scope.conv_nl(conv4, name="conv4_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv4_1']) conv5 = nd.scope.conv_nl(conv4_1, name="conv5", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv5']) conv5_1 = nd.scope.conv_nl(conv5, name="conv5_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv5_1']) conv6 = nd.scope.conv_nl(conv5_1, name="conv6", kernel_size=3, stride=2, pad=1, num_output=self._encoder_channels['conv6']) conv6_1 = nd.scope.conv_nl(conv6, name="conv6_1", kernel_size=3, stride=1, pad=1, num_output=self._encoder_channels['conv6_1']) prediction6 = self.predict(conv6_1, level=6, loss_weight=self._loss_weights['level6'], out=out) with nd.Scope('decoder'): decoder5, prediction5 = \ self.refine(level=5, input=conv6_1, input_prediction=prediction6, features=conv5_1, out=out) if self._exit_after == 5: out.final = out.levels[5] return out decoder4, prediction4 = \ self.refine(level=4, input=decoder5, input_prediction=prediction5, features=conv4_1, out=out) if self._exit_after == 4: out.final = out.levels[4] return out decoder3, prediction3 = \ self.refine(level=3, input=decoder4, input_prediction=prediction4, features=conv3_1, out=out) if self._exit_after == 3: out.final = out.levels[3] return out decoder2, prediction2 = \ self.refine(level=2, input=decoder3, input_prediction=prediction3, features=input.conv2a, out=out) if self._exit_after == 2: out.final = out.levels[2] return out decoder1, prediction1 = \ self.refine(level=1, input=decoder2, input_prediction=prediction2, features=input.conv1a, out=out) if self._exit_after == 1: out.final = out.levels[1] return out if edge_features is None: raise BaseException('Architecture_S needs edge features if now shallow') edges = nd.scope.conv_nl(edge_features, name="conv_edges", kernel_size=3, stride=1, pad=1, num_output=self._decoder_channels['level0']) decoder0, prediction0 = \ self.refine(level=0, input=decoder1, input_prediction=prediction1, features=edges, out=out) out.final = out.levels[0] return out