def reset(self): if self._shuffle: logger.debug('Shuffle start.') self._order = list(self._rng.permutation(list(range(self._size)))) logger.debug('Shuffle end.') self._generation += 1 super(CsvDataSource, self).reset()
def reset(self): if self._shuffle: logger.debug('Shuffle start.') self._order = list( numpy.random.permutation(list(range(self._size)))) logger.debug('Shuffle end.') self._generation += 1 super(CsvDataSource, self).reset()
def _network(proto, default_context, batch_size, all_variables): network = Network() network.name = proto.name # Read Repeat Info network.repeat_info = {} for r in proto.repeat_info: network.repeat_info[r.id] = r.times network.variables = OrderedDict() if batch_size is None: network.batch_size = proto.batch_size else: network.batch_size = batch_size for v in proto.variable: for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in v.repeat_id]))): name = v.name + ''.join(['_' + v.repeat_id[index] + '[' + str(i) + ']' for index, i in enumerate(variable_index)]) if name in all_variables: variable = all_variables[name] else: shape = tuple( [d if d >= 1 else network.batch_size for d in v.shape.dim]) variable = _create_variable(v, name, shape) all_variables[name] = variable network.variables[name] = variable logger.debug('{}'.format( (name, variable.shape, v.initializer.type if v.initializer.type else '-', v.initializer.multiplier))) network.functions = OrderedDict() network.function_inputs = OrderedDict() network.function_outputs = OrderedDict() network.variable_inputs = OrderedDict() network.variable_outputs = OrderedDict() for f in proto.function: ctx = default_context if f.context.backend == "" else context( f.context) for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in f.repeat_id]))): function, input_variable_names, output_variable_names = _create_function( ctx, network, f, variable_index) if function is not None: network.functions[function.name] = function for v_name in output_variable_names: network.variable_inputs[ network.variables[v_name]] = [function] for v_name in input_variable_names: if not network.variables[v_name] in network.variable_outputs: network.variable_outputs[ network.variables[v_name]] = [] network.variable_outputs[ network.variables[v_name]].append(function) network.setup(optimize=True) return network
def _network(proto, default_context, all_variables): network = Network() network.name = proto.name # Read Repeat Info network.repeat_info = {} for r in proto.repeat_info: network.repeat_info[r.id] = r.times network.variables = OrderedDict() network.batch_size = proto.batch_size for v in proto.variable: for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in v.repeat_id]))): name = v.name + ''.join(['_' + v.repeat_id[index] + '[' + str(i) + ']' for index, i in enumerate(variable_index)]) if name in all_variables: variable = all_variables[name] else: shape = tuple( [d if d >= 1 else network.batch_size for d in v.shape.dim]) variable = _create_variable(v, name, shape) all_variables[name] = variable network.variables[name] = variable logger.debug('{}'.format( (name, variable.shape, v.initializer.type if v.initializer.type else '-', v.initializer.multiplier))) network.functions = OrderedDict() network.function_inputs = OrderedDict() network.function_outputs = OrderedDict() network.variable_inputs = OrderedDict() network.variable_outputs = OrderedDict() for f in proto.function: ctx = default_context if f.context.backend == "" else context( f.context) for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in f.repeat_id]))): function, input_variable_names, output_variable_names = _create_function( ctx, network, f, variable_index) if function is not None: network.functions[function.name] = function for v_name in output_variable_names: network.variable_inputs[ network.variables[v_name]] = [function] for v_name in input_variable_names: if not network.variables[v_name] in network.variable_outputs: network.variable_outputs[ network.variables[v_name]] = [] network.variable_outputs[ network.variables[v_name]].append(function) network.setup(optimize=True) return network
def resnet_imagenet(x, num_classes, num_layers, shortcut_type, test, tiny=False): """ Args: x : Variable num_classes : Number of classes of outputs num_layers : Number of layers of ResNet chosen from (18, 34, 50, 101, 152) shortcut_type : 'c', 'b', '' 'c' : Use Convolution anytime 'b' : Use Convolution if numbers of channels of input and output mismatch. '' : Use Identity mapping if channels match, otherwise zero padding. test : Construct net for testing. tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56). """ layers = { 18: ((2, 2, 2, 2), basicblock, 1), 34: ((3, 4, 6, 3), basicblock, 1), 50: ((3, 4, 6, 3), bottleneck, 4), 101: ((3, 4, 23, 3), bottleneck, 4), 152: ((3, 8, 36, 3), bottleneck, 4) } counts, block, ocoef = layers[num_layers] logger.debug(x.shape) with nn.parameter_scope("conv1"): stride = (1, 1) if tiny else (2, 2) r = PF.convolution(x, 64, (7, 7), pad=(3, 3), stride=stride, with_bias=False) r = F.relu(PF.batch_normalization(r, batch_stat=not test), inplace=True) r = F.max_pooling(r, (3, 3), stride, pad=(1, 1)) hidden = {} hidden['r0'] = r ochannels = [64, 128, 256, 512] strides = [1, 2, 2, 2] logger.debug(r.shape) for i in range(4): with nn.parameter_scope("res{}".format(i + 1)): r = layer(r, block, ochannels[i] * ocoef, counts[i], (strides[i], strides[i]), shortcut_type, test) hidden['r{}'.format(i + 1)] = r logger.debug(r.shape) r = F.average_pooling(r, r.shape[-2:]) with nn.parameter_scope("fc"): r = PF.affine(r, num_classes) logger.debug(r.shape) return r, hidden
def resnet_cifar10(x, num_classes, cfg, test): """ Args: x : Variable num_classes : Number of classes of outputs cfg : network configuration """ layers = { 20: ((3, 3, 3), basicblock, 1), 32: ((5, 5, 5), basicblock, 1), 44: ((7, 7, 7), basicblock, 1), 56: ((9, 9, 9), basicblock, 1), 110: ((18, 18, 18), basicblock, 1) } counts, block, ocoef = layers[cfg.num_layers] logger.debug(x.shape) with nn.parameter_scope("conv1"): stride = (1, 1) r = conv(x, 16, (3, 3), cfg, test, pad=(1, 1), stride=stride, with_bias=False) r = nonl(PF.batch_normalization(r, batch_stat=not test), cfg, inplace=True) hidden = {} hidden['r0'] = r ochannels = [16, 32, 64] strides = [1, 2, 2] logger.debug(r.shape) for i in range(3): with nn.parameter_scope("res{}".format(i + 1)): r = layer(r, block, ochannels[i] * ocoef, counts[i], (strides[i], strides[i]), cfg, test) hidden['r{}'.format(i + 1)] = r logger.debug(r.shape) r = F.average_pooling(r, r.shape[-2:]) with nn.parameter_scope("fc"): stdv = 1. / np.sqrt(np.prod(r.shape[1:])) init = nn.initializer.UniformInitializer(lim=(-stdv, stdv)) r = affi(r, num_classes, cfg, test, w_init=init, b_init=init) if cfg.scale_layer: s = get_parameter_or_create('scale_layer', shape=(1, 1), initializer=np.ones((1, 1), dtype=np.float32), need_grad=True) r = s * r logger.debug(r.shape) return r, hidden
def __call__(self, pf): if pf.type == 'Identity': if self.is_required: if not self.is_required(pf): return proto_network = pf.owner() input_name = None need_grad = False for pv_name in pf.inputs: pv = proto_network.variables[pv_name] \ if pv_name in proto_network.variables \ else proto_network.parameters[pv_name] parent = pv.parent required = filter(lambda k: proto_network.functions[k] != pf, pv.required) input_name = pv_name need_grad = pv.need_grad break for pv_name in pf.outputs: pv = proto_network.variables[pv_name] pv.parent = parent for r in pv.required: r_func = proto_network.functions[r] index = r_func.inputs.index(pv.name) r_func.inputs[index] = input_name pv.required += required if pv_name in proto_network.outputs: index = proto_network.outputs.index(pv_name) proto_network.outputs[index] = input_name self.renamed[pv.name] = input_name pv.name = input_name pv.need_grad = need_grad proto_network.variables[input_name] = pv del proto_network.variables[pv_name] break del proto_network.functions[pf.name] logger.debug( "proto_variable:{} is deleted, proto_function:{} is deleted!". format(pv.name, pf.name))
def setup(self, optimize=False): if optimize: for func in self.functions.values(): # remove identity layer if func.function_instance.name[0:8] == "Identity": assert(len(func.inputs) == 1) assert(len(func.outputs) == 1) # if the identity function is not terminal (keep terminal # identity function) if func.outputs[0] in self.variable_outputs: next_functions = self.variable_outputs[func.outputs[0]] self.variable_outputs[func.inputs[0]].remove(func) self.variable_outputs[ func.inputs[0]].extend(next_functions) for next_function in next_functions: next_function.inputs = [func.inputs[0] if v == func.outputs[ 0] else v for v in next_function.inputs] del self.functions[func.name] del self.variables[func.outputs[0].name] for func in self.functions.values(): if func.function_instance.inplace_data(0) > 0 and func.function_instance.inplace_grad(0) > 0: func.outputs[0].variable_instance = func.inputs[ 0].variable_instance # create variable instances for variable in self.variables.values(): if variable.variable_instance.shape != variable.shape: if hasattr(variable.variable_instance, 'reset_shape'): variable.variable_instance.reset_shape( variable.shape, force=True) else: variable.variable_instance.reshape( variable.shape, force=True) # setup functions for i, func in enumerate(self.functions.values()): func.variable_inputs = [v.variable_instance for v in func.inputs] func.variable_outputs = [v.variable_instance for v in func.outputs] try: self.setup_function(func) except: print_network_traceback(self.functions.values()[ min(0, i - 4):i + 1]) raise # set link structure to each layer from itertools import chain for func in self.functions.values(): func.input_functions = list(chain.from_iterable( [self.variable_inputs[v] for v in func.inputs if v in self.variable_inputs])) func.output_functions = list(chain.from_iterable( [self.variable_outputs[v] for v in func.outputs if v in self.variable_outputs])) logger.debug(func.name) logger.debug(' in: {}'.format( [f.name for f in func.input_functions])) logger.debug(' out: {}'.format( [f.name for f in func.output_functions]))
def __call__(self, x): ''' Defines a ResNet-like network according to the configuration specified. Args: x: A Variable object which has a shape with a format `NCHW` if `channel_last=False` else `NHWC`. Returns: * An output `Variable` of classification layer * Intermediate `Variable` outputs from input and output of each cell ''' logger.debug(x.shape) # First convolution axes = [get_channel_axis(self.channel_last)] with nn.parameter_scope("conv1"): r = pf_convolution(x, 64, (7, 7), stride=(2, 2), channel_last=self.channel_last) r = PF.fused_batch_normalization(r, axes=axes, batch_stat=not self.test) mp_opts = dict( ignore_border=False) if self.max_pooling_ceil_border else dict( pad=(1, 1)) r = F.max_pooling(r, (3, 3), (2, 2), channel_last=self.channel_last, **mp_opts) hidden = {} hidden['r0'] = r logger.debug(r.shape) # Create cells each of which consists of blocks repeatedly applied cell_configs = self.get_cell_configurations(self.num_layers) for i, (counts, ochannels, strides) in enumerate(zip(*cell_configs)): with nn.parameter_scope("res{}".format(i + 1)): r = self.cell(r, ochannels, counts, (strides, ) * 2) hidden['r{}'.format(i + 1)] = r logger.debug(r.shape) # Global average pooling pool_shape = get_spatial_shape(r.shape, self.channel_last) r = F.average_pooling(r, pool_shape, channel_last=self.channel_last) # Final classification layer with nn.parameter_scope("fc"): r = pf_affine(r, self.num_classes, channel_last=self.channel_last) return r, hidden
def setup(self, optimize=False): if optimize: for func in list(self.functions.values()): # remove identity layer if func.function_instance.name[0:8] == "Identity" and not func.persistent: assert (len(func.inputs) == 1) assert (len(func.outputs) == 1) # if the identity function is not terminal (keep terminal # identity function) if func.outputs[0] in self.variable_outputs: next_functions = self.variable_outputs[func.outputs[0]] self.variable_outputs[func.inputs[0]].remove(func) self.variable_outputs[ func.inputs[0]].extend(next_functions) for next_function in next_functions: next_function.inputs = [func.inputs[0] if v == func.outputs[ 0] else v for v in next_function.inputs] del self.functions[func.name] del self.variables[func.outputs[0].name] # create variable instances for variable in self.variables.values(): if variable.variable_instance.shape != variable.shape: if hasattr(variable.variable_instance, 'reset_shape'): variable.variable_instance.reset_shape( variable.shape, force=True) else: variable.variable_instance.reshape( variable.shape, force=True) # setup functions for i, func in enumerate(self.functions.values()): func.variable_inputs = [v.variable_instance for v in func.inputs] func.variable_outputs = [v.variable_instance for v in func.outputs] try: self.setup_function(func) except: print_network_traceback(list(self.functions.values())[ max(0, i - 4):i + 1]) raise # set link structure to each layer from itertools import chain for func in self.functions.values(): func.input_functions = list(chain.from_iterable( [self.variable_inputs[v] for v in func.inputs if v in self.variable_inputs])) func.output_functions = list(chain.from_iterable( [self.variable_outputs[v] for v in func.outputs if v in self.variable_outputs])) logger.debug(func.name) logger.debug(' in: {}'.format( [f.name for f in func.input_functions])) logger.debug(' out: {}'.format( [f.name for f in func.output_functions]))
'--max_epoch', type=int, default=3, help='Max epoch to read.') parser.add_argument('-w', '--wait', type=float, default=0, help='Wait time for dummy data processing.') parser.add_argument('uri', help='PATH to CSV_DATASET format file or ' '"MNIST_TRAIN", "MNIST_TEST", "TINY_IMAGENET_TRAIN",' '"TINY_IMAGENET_VAL"') args = parser.parse_args() logger.debug('memory_cache: {}'.format(args.memory_cache)) logger.debug('file_cache: {}'.format(args.file_cache)) logger.debug('shuffle: {}'.format(args.shuffle)) logger.debug('batch_size: {}'.format(args.batch_size)) logger.debug('cache_size: {}'.format(args.cache_size)) logger.debug('memory_size: {}'.format(args.memory_size)) logger.debug('output: {}'.format(args.output)) logger.debug('normalize: {}'.format(args.normalize)) logger.debug('max_epoch: {}'.format(args.max_epoch)) logger.debug('wait: {}'.format(args.wait)) nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size', '{}'.format(args.cache_size)) nnabla_config.set('DATA_ITERATOR', 'data_source_buffer_max_size', '{}'.format(args.memory_size))
'--file_cache', action='store_true', help='Use file cache') parser.add_argument('-s', '--cache_size', type=int, default=100, help='Cache size (num of data).') parser.add_argument('-o', '--output', type=str, default='cache', help='If specified, cache data will output to here.') args = parser.parse_args() logger.debug('file_cache: {}'.format(args.file_cache)) logger.debug('cache_size: {}'.format(args.cache_size)) logger.debug('output: {}'.format(args.output)) nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size', '{}'.format(args.cache_size)) nnabla_config.set('DATA_ITERATOR', 'cache_file_format', '.h5') HERE = os.path.dirname(__file__) nnabla_examples_root = os.path.join(HERE, '../../../../nnabla-examples') mnist_examples_root = os.path.realpath( os.path.join(nnabla_examples_root, 'mnist-collection')) sys.path.append(mnist_examples_root) from mnist_data import MnistDataSource mnist_training_cache = args.output + '/mnist_training.cache'
def get_renamed(str): ret_str = str.replace(']', '').replace('[', '_') if ret_str != str: logger.debug("{} --> {}".format(str, ret_str)) return ret_str
def dla_imagenet(x, num_classes, num_layers, test, residual_root=False, tiny=False, channel_last=False): """ Args: x : Variable num_classes : Number of classes of outputs num_layers : Number of layers of DLA chosen from (34). test : Construct net for testing. tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56). """ layers = { # 18: ((2, 2, 2, 2), basicblock, 1), 34: ((1, 1, 1, 2, 2, 1), (False, False, False, True, True, True), basicblock) # 50: ((3, 4, 6, 3), bottleneck, 4), # 101: ((3, 4, 23, 3), bottleneck, 4), # 152: ((3, 8, 36, 3), bottleneck, 4) } ochannels = [16, 32, 64, 128, 256, 512] levels, levels_root, block = layers[num_layers] strides = [1, 2, 2, 2, 2, 2] logger.debug(x.shape) axes = 3 if channel_last else 1 with nn.parameter_scope("conv1"): stride = (1, 1) r = pf_convolution(x, 16, (7, 7), pad=(3, 3), stride=stride, with_bias=False, channel_last=channel_last) r = F.relu(PF.batch_normalization(r, axes=[axes], batch_stat=not test)) hidden = {} hidden['conv0'] = r logger.debug(r.shape) with nn.parameter_scope("level0"): r = _make_conv_level(r, ochannels[0], levels[0], test=test, stride=strides[0], channel_last=channel_last) hidden['level0'] = r logger.debug(r.shape) with nn.parameter_scope("level1"): r = _make_conv_level(r, ochannels[1], levels[1], test=test, stride=strides[1], channel_last=channel_last) hidden['level1'] = r logger.debug(r.shape) with nn.parameter_scope("level2"): r, _ = _make_tree_level1(r, None, block, ochannels[2], levels[2], test, levels_root[2], stride=strides[2], channel_last=channel_last) hidden['level2'] = r logger.debug(r.shape) with nn.parameter_scope("level3"): r = _make_tree_level2(r, None, block, ochannels[3], levels[3], test, levels_root[3], stride=strides[3], channel_last=channel_last) hidden['level3'] = r logger.debug(r.shape) with nn.parameter_scope("level4"): r = _make_tree_level2(r, None, block, ochannels[4], levels[4], test, levels_root[4], stride=strides[4], channel_last=channel_last) hidden['level4'] = r logger.debug(r.shape) with nn.parameter_scope("level5"): r, _ = _make_tree_level1(r, None, block, ochannels[5], levels[5], test, levels_root[5], stride=strides[5], channel_last=channel_last) hidden['level5'] = r logger.debug(r.shape) pool_shape = r.shape[-2:] if channel_last: pool_shape = r.shape[1:3] r = F.average_pooling(r, pool_shape, channel_last=channel_last) with nn.parameter_scope("fc"): r = pf_affine(r, num_classes, channel_last=channel_last) logger.debug(r.shape) return r, hidden