def reset(self):
if self._shuffle:
logger.debug('Shuffle start.')
self._order = list(self._rng.permutation(list(range(self._size))))
logger.debug('Shuffle end.')
self._generation += 1
super(CsvDataSource, self).reset()
def reset(self):
if self._shuffle:
logger.debug('Shuffle start.')
self._order = list(
numpy.random.permutation(list(range(self._size))))
logger.debug('Shuffle end.')
self._generation += 1
super(CsvDataSource, self).reset()
def _network(proto, default_context, batch_size, all_variables):
network = Network()
network.name = proto.name
# Read Repeat Info
network.repeat_info = {}
for r in proto.repeat_info:
network.repeat_info[r.id] = r.times
network.variables = OrderedDict()
if batch_size is None:
network.batch_size = proto.batch_size
else:
network.batch_size = batch_size
for v in proto.variable:
for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in v.repeat_id]))):
name = v.name + ''.join(['_' + v.repeat_id[index] + '[' +
str(i) + ']' for index, i in enumerate(variable_index)])
if name in all_variables:
variable = all_variables[name]
else:
shape = tuple(
[d if d >= 1 else network.batch_size for d in v.shape.dim])
variable = _create_variable(v, name, shape)
all_variables[name] = variable
network.variables[name] = variable
logger.debug('{}'.format(
(name, variable.shape, v.initializer.type if v.initializer.type else '-', v.initializer.multiplier)))
network.functions = OrderedDict()
network.function_inputs = OrderedDict()
network.function_outputs = OrderedDict()
network.variable_inputs = OrderedDict()
network.variable_outputs = OrderedDict()
for f in proto.function:
ctx = default_context if f.context.backend == "" else context(
f.context)
for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in f.repeat_id]))):
function, input_variable_names, output_variable_names = _create_function(
ctx, network, f, variable_index)
if function is not None:
network.functions[function.name] = function
for v_name in output_variable_names:
network.variable_inputs[
network.variables[v_name]] = [function]
for v_name in input_variable_names:
if not network.variables[v_name] in network.variable_outputs:
network.variable_outputs[
network.variables[v_name]] = []
network.variable_outputs[
network.variables[v_name]].append(function)
network.setup(optimize=True)
return network
def _network(proto, default_context, all_variables):
network = Network()
network.name = proto.name
# Read Repeat Info
network.repeat_info = {}
for r in proto.repeat_info:
network.repeat_info[r.id] = r.times
network.variables = OrderedDict()
network.batch_size = proto.batch_size
for v in proto.variable:
for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in v.repeat_id]))):
name = v.name + ''.join(['_' + v.repeat_id[index] + '[' +
str(i) + ']' for index, i in enumerate(variable_index)])
if name in all_variables:
variable = all_variables[name]
else:
shape = tuple(
[d if d >= 1 else network.batch_size for d in v.shape.dim])
variable = _create_variable(v, name, shape)
all_variables[name] = variable
network.variables[name] = variable
logger.debug('{}'.format(
(name, variable.shape, v.initializer.type if v.initializer.type else '-', v.initializer.multiplier)))
network.functions = OrderedDict()
network.function_inputs = OrderedDict()
network.function_outputs = OrderedDict()
network.variable_inputs = OrderedDict()
network.variable_outputs = OrderedDict()
for f in proto.function:
ctx = default_context if f.context.backend == "" else context(
f.context)
for variable_index in itertools.product(*map(tuple, map(range, [network.repeat_info[id] for id in f.repeat_id]))):
function, input_variable_names, output_variable_names = _create_function(
ctx, network, f, variable_index)
if function is not None:
network.functions[function.name] = function
for v_name in output_variable_names:
network.variable_inputs[
network.variables[v_name]] = [function]
for v_name in input_variable_names:
if not network.variables[v_name] in network.variable_outputs:
network.variable_outputs[
network.variables[v_name]] = []
network.variable_outputs[
network.variables[v_name]].append(function)
network.setup(optimize=True)
return network
def resnet_imagenet(x,
num_classes,
num_layers,
shortcut_type,
test,
tiny=False):
"""
Args:
x : Variable
num_classes : Number of classes of outputs
num_layers : Number of layers of ResNet chosen from (18, 34, 50, 101, 152)
shortcut_type : 'c', 'b', ''
'c' : Use Convolution anytime
'b' : Use Convolution if numbers of channels of input
and output mismatch.
'' : Use Identity mapping if channels match, otherwise zero padding.
test : Construct net for testing.
tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56).
"""
layers = {
18: ((2, 2, 2, 2), basicblock, 1),
34: ((3, 4, 6, 3), basicblock, 1),
50: ((3, 4, 6, 3), bottleneck, 4),
101: ((3, 4, 23, 3), bottleneck, 4),
152: ((3, 8, 36, 3), bottleneck, 4)
}
counts, block, ocoef = layers[num_layers]
logger.debug(x.shape)
with nn.parameter_scope("conv1"):
stride = (1, 1) if tiny else (2, 2)
r = PF.convolution(x,
64, (7, 7),
pad=(3, 3),
stride=stride,
with_bias=False)
r = F.relu(PF.batch_normalization(r, batch_stat=not test),
inplace=True)
r = F.max_pooling(r, (3, 3), stride, pad=(1, 1))
hidden = {}
hidden['r0'] = r
ochannels = [64, 128, 256, 512]
strides = [1, 2, 2, 2]
logger.debug(r.shape)
for i in range(4):
with nn.parameter_scope("res{}".format(i + 1)):
r = layer(r, block, ochannels[i] * ocoef, counts[i],
(strides[i], strides[i]), shortcut_type, test)
hidden['r{}'.format(i + 1)] = r
logger.debug(r.shape)
r = F.average_pooling(r, r.shape[-2:])
with nn.parameter_scope("fc"):
r = PF.affine(r, num_classes)
logger.debug(r.shape)
return r, hidden
def resnet_cifar10(x, num_classes, cfg, test):
"""
Args:
x : Variable
num_classes : Number of classes of outputs
cfg : network configuration
"""
layers = {
20: ((3, 3, 3), basicblock, 1),
32: ((5, 5, 5), basicblock, 1),
44: ((7, 7, 7), basicblock, 1),
56: ((9, 9, 9), basicblock, 1),
110: ((18, 18, 18), basicblock, 1)
}
counts, block, ocoef = layers[cfg.num_layers]
logger.debug(x.shape)
with nn.parameter_scope("conv1"):
stride = (1, 1)
r = conv(x,
16, (3, 3),
cfg,
test,
pad=(1, 1),
stride=stride,
with_bias=False)
r = nonl(PF.batch_normalization(r, batch_stat=not test),
cfg,
inplace=True)
hidden = {}
hidden['r0'] = r
ochannels = [16, 32, 64]
strides = [1, 2, 2]
logger.debug(r.shape)
for i in range(3):
with nn.parameter_scope("res{}".format(i + 1)):
r = layer(r, block, ochannels[i] * ocoef, counts[i],
(strides[i], strides[i]), cfg, test)
hidden['r{}'.format(i + 1)] = r
logger.debug(r.shape)
r = F.average_pooling(r, r.shape[-2:])
with nn.parameter_scope("fc"):
stdv = 1. / np.sqrt(np.prod(r.shape[1:]))
init = nn.initializer.UniformInitializer(lim=(-stdv, stdv))
r = affi(r, num_classes, cfg, test, w_init=init, b_init=init)
if cfg.scale_layer:
s = get_parameter_or_create('scale_layer',
shape=(1, 1),
initializer=np.ones((1, 1),
dtype=np.float32),
need_grad=True)
r = s * r
logger.debug(r.shape)
return r, hidden
def __call__(self, pf):
if pf.type == 'Identity':
if self.is_required:
if not self.is_required(pf):
return
proto_network = pf.owner()
input_name = None
need_grad = False
for pv_name in pf.inputs:
pv = proto_network.variables[pv_name] \
if pv_name in proto_network.variables \
else proto_network.parameters[pv_name]
parent = pv.parent
required = filter(lambda k: proto_network.functions[k] != pf,
pv.required)
input_name = pv_name
need_grad = pv.need_grad
break
for pv_name in pf.outputs:
pv = proto_network.variables[pv_name]
pv.parent = parent
for r in pv.required:
r_func = proto_network.functions[r]
index = r_func.inputs.index(pv.name)
r_func.inputs[index] = input_name
pv.required += required
if pv_name in proto_network.outputs:
index = proto_network.outputs.index(pv_name)
proto_network.outputs[index] = input_name
self.renamed[pv.name] = input_name
pv.name = input_name
pv.need_grad = need_grad
proto_network.variables[input_name] = pv
del proto_network.variables[pv_name]
break
del proto_network.functions[pf.name]
logger.debug(
"proto_variable:{} is deleted, proto_function:{} is deleted!".
format(pv.name, pf.name))
def setup(self, optimize=False):
if optimize:
for func in self.functions.values():
# remove identity layer
if func.function_instance.name[0:8] == "Identity":
assert(len(func.inputs) == 1)
assert(len(func.outputs) == 1)
# if the identity function is not terminal (keep terminal
# identity function)
if func.outputs[0] in self.variable_outputs:
next_functions = self.variable_outputs[func.outputs[0]]
self.variable_outputs[func.inputs[0]].remove(func)
self.variable_outputs[
func.inputs[0]].extend(next_functions)
for next_function in next_functions:
next_function.inputs = [func.inputs[0] if v == func.outputs[
0] else v for v in next_function.inputs]
del self.functions[func.name]
del self.variables[func.outputs[0].name]
for func in self.functions.values():
if func.function_instance.inplace_data(0) > 0 and func.function_instance.inplace_grad(0) > 0:
func.outputs[0].variable_instance = func.inputs[
0].variable_instance
# create variable instances
for variable in self.variables.values():
if variable.variable_instance.shape != variable.shape:
if hasattr(variable.variable_instance, 'reset_shape'):
variable.variable_instance.reset_shape(
variable.shape, force=True)
else:
variable.variable_instance.reshape(
variable.shape, force=True)
# setup functions
for i, func in enumerate(self.functions.values()):
func.variable_inputs = [v.variable_instance for v in func.inputs]
func.variable_outputs = [v.variable_instance for v in func.outputs]
try:
self.setup_function(func)
except:
print_network_traceback(self.functions.values()[
min(0, i - 4):i + 1])
raise
# set link structure to each layer
from itertools import chain
for func in self.functions.values():
func.input_functions = list(chain.from_iterable(
[self.variable_inputs[v] for v in func.inputs if v in self.variable_inputs]))
func.output_functions = list(chain.from_iterable(
[self.variable_outputs[v] for v in func.outputs if v in self.variable_outputs]))
logger.debug(func.name)
logger.debug(' in: {}'.format(
[f.name for f in func.input_functions]))
logger.debug(' out: {}'.format(
[f.name for f in func.output_functions]))
def __call__(self, x):
'''
Defines a ResNet-like network according to the configuration specified.
Args:
x:
A Variable object which has a shape with a format
`NCHW` if `channel_last=False` else `NHWC`.
Returns:
* An output `Variable` of classification layer
* Intermediate `Variable` outputs from input and output of each
cell
'''
logger.debug(x.shape)
# First convolution
axes = [get_channel_axis(self.channel_last)]
with nn.parameter_scope("conv1"):
r = pf_convolution(x,
64, (7, 7),
stride=(2, 2),
channel_last=self.channel_last)
r = PF.fused_batch_normalization(r,
axes=axes,
batch_stat=not self.test)
mp_opts = dict(
ignore_border=False) if self.max_pooling_ceil_border else dict(
pad=(1, 1))
r = F.max_pooling(r, (3, 3), (2, 2),
channel_last=self.channel_last,
**mp_opts)
hidden = {}
hidden['r0'] = r
logger.debug(r.shape)
# Create cells each of which consists of blocks repeatedly applied
cell_configs = self.get_cell_configurations(self.num_layers)
for i, (counts, ochannels, strides) in enumerate(zip(*cell_configs)):
with nn.parameter_scope("res{}".format(i + 1)):
r = self.cell(r, ochannels, counts, (strides, ) * 2)
hidden['r{}'.format(i + 1)] = r
logger.debug(r.shape)
# Global average pooling
pool_shape = get_spatial_shape(r.shape, self.channel_last)
r = F.average_pooling(r, pool_shape, channel_last=self.channel_last)
# Final classification layer
with nn.parameter_scope("fc"):
r = pf_affine(r, self.num_classes, channel_last=self.channel_last)
return r, hidden
def setup(self, optimize=False):
if optimize:
for func in list(self.functions.values()):
# remove identity layer
if func.function_instance.name[0:8] == "Identity" and not func.persistent:
assert (len(func.inputs) == 1)
assert (len(func.outputs) == 1)
# if the identity function is not terminal (keep terminal
# identity function)
if func.outputs[0] in self.variable_outputs:
next_functions = self.variable_outputs[func.outputs[0]]
self.variable_outputs[func.inputs[0]].remove(func)
self.variable_outputs[
func.inputs[0]].extend(next_functions)
for next_function in next_functions:
next_function.inputs = [func.inputs[0] if v == func.outputs[
0] else v for v in next_function.inputs]
del self.functions[func.name]
del self.variables[func.outputs[0].name]
# create variable instances
for variable in self.variables.values():
if variable.variable_instance.shape != variable.shape:
if hasattr(variable.variable_instance, 'reset_shape'):
variable.variable_instance.reset_shape(
variable.shape, force=True)
else:
variable.variable_instance.reshape(
variable.shape, force=True)
# setup functions
for i, func in enumerate(self.functions.values()):
func.variable_inputs = [v.variable_instance for v in func.inputs]
func.variable_outputs = [v.variable_instance for v in func.outputs]
try:
self.setup_function(func)
except:
print_network_traceback(list(self.functions.values())[
max(0, i - 4):i + 1])
raise
# set link structure to each layer
from itertools import chain
for func in self.functions.values():
func.input_functions = list(chain.from_iterable(
[self.variable_inputs[v] for v in func.inputs if v in self.variable_inputs]))
func.output_functions = list(chain.from_iterable(
[self.variable_outputs[v] for v in func.outputs if v in self.variable_outputs]))
logger.debug(func.name)
logger.debug(' in: {}'.format(
[f.name for f in func.input_functions]))
logger.debug(' out: {}'.format(
[f.name for f in func.output_functions]))
'--max_epoch',
type=int,
default=3,
help='Max epoch to read.')
parser.add_argument('-w',
'--wait',
type=float,
default=0,
help='Wait time for dummy data processing.')
parser.add_argument('uri',
help='PATH to CSV_DATASET format file or '
'"MNIST_TRAIN", "MNIST_TEST", "TINY_IMAGENET_TRAIN",'
'"TINY_IMAGENET_VAL"')
args = parser.parse_args()
logger.debug('memory_cache: {}'.format(args.memory_cache))
logger.debug('file_cache: {}'.format(args.file_cache))
logger.debug('shuffle: {}'.format(args.shuffle))
logger.debug('batch_size: {}'.format(args.batch_size))
logger.debug('cache_size: {}'.format(args.cache_size))
logger.debug('memory_size: {}'.format(args.memory_size))
logger.debug('output: {}'.format(args.output))
logger.debug('normalize: {}'.format(args.normalize))
logger.debug('max_epoch: {}'.format(args.max_epoch))
logger.debug('wait: {}'.format(args.wait))
nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size',
'{}'.format(args.cache_size))
nnabla_config.set('DATA_ITERATOR', 'data_source_buffer_max_size',
'{}'.format(args.memory_size))
'--file_cache',
action='store_true',
help='Use file cache')
parser.add_argument('-s',
'--cache_size',
type=int,
default=100,
help='Cache size (num of data).')
parser.add_argument('-o',
'--output',
type=str,
default='cache',
help='If specified, cache data will output to here.')
args = parser.parse_args()
logger.debug('file_cache: {}'.format(args.file_cache))
logger.debug('cache_size: {}'.format(args.cache_size))
logger.debug('output: {}'.format(args.output))
nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size',
'{}'.format(args.cache_size))
nnabla_config.set('DATA_ITERATOR', 'cache_file_format', '.h5')
HERE = os.path.dirname(__file__)
nnabla_examples_root = os.path.join(HERE, '../../../../nnabla-examples')
mnist_examples_root = os.path.realpath(
os.path.join(nnabla_examples_root, 'mnist-collection'))
sys.path.append(mnist_examples_root)
from mnist_data import MnistDataSource
mnist_training_cache = args.output + '/mnist_training.cache'
def get_renamed(str):
ret_str = str.replace(']', '').replace('[', '_')
if ret_str != str:
logger.debug("{} --> {}".format(str, ret_str))
return ret_str
def dla_imagenet(x,
num_classes,
num_layers,
test,
residual_root=False,
tiny=False,
channel_last=False):
"""
Args:
x : Variable
num_classes : Number of classes of outputs
num_layers : Number of layers of DLA chosen from (34).
test : Construct net for testing.
tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56).
"""
layers = {
# 18: ((2, 2, 2, 2), basicblock, 1),
34: ((1, 1, 1, 2, 2, 1), (False, False, False, True, True, True),
basicblock)
# 50: ((3, 4, 6, 3), bottleneck, 4),
# 101: ((3, 4, 23, 3), bottleneck, 4),
# 152: ((3, 8, 36, 3), bottleneck, 4)
}
ochannels = [16, 32, 64, 128, 256, 512]
levels, levels_root, block = layers[num_layers]
strides = [1, 2, 2, 2, 2, 2]
logger.debug(x.shape)
axes = 3 if channel_last else 1
with nn.parameter_scope("conv1"):
stride = (1, 1)
r = pf_convolution(x,
16, (7, 7),
pad=(3, 3),
stride=stride,
with_bias=False,
channel_last=channel_last)
r = F.relu(PF.batch_normalization(r, axes=[axes], batch_stat=not test))
hidden = {}
hidden['conv0'] = r
logger.debug(r.shape)
with nn.parameter_scope("level0"):
r = _make_conv_level(r,
ochannels[0],
levels[0],
test=test,
stride=strides[0],
channel_last=channel_last)
hidden['level0'] = r
logger.debug(r.shape)
with nn.parameter_scope("level1"):
r = _make_conv_level(r,
ochannels[1],
levels[1],
test=test,
stride=strides[1],
channel_last=channel_last)
hidden['level1'] = r
logger.debug(r.shape)
with nn.parameter_scope("level2"):
r, _ = _make_tree_level1(r,
None,
block,
ochannels[2],
levels[2],
test,
levels_root[2],
stride=strides[2],
channel_last=channel_last)
hidden['level2'] = r
logger.debug(r.shape)
with nn.parameter_scope("level3"):
r = _make_tree_level2(r,
None,
block,
ochannels[3],
levels[3],
test,
levels_root[3],
stride=strides[3],
channel_last=channel_last)
hidden['level3'] = r
logger.debug(r.shape)
with nn.parameter_scope("level4"):
r = _make_tree_level2(r,
None,
block,
ochannels[4],
levels[4],
test,
levels_root[4],
stride=strides[4],
channel_last=channel_last)
hidden['level4'] = r
logger.debug(r.shape)
with nn.parameter_scope("level5"):
r, _ = _make_tree_level1(r,
None,
block,
ochannels[5],
levels[5],
test,
levels_root[5],
stride=strides[5],
channel_last=channel_last)
hidden['level5'] = r
logger.debug(r.shape)
pool_shape = r.shape[-2:]
if channel_last:
pool_shape = r.shape[1:3]
r = F.average_pooling(r, pool_shape, channel_last=channel_last)
with nn.parameter_scope("fc"):
r = pf_affine(r, num_classes, channel_last=channel_last)
logger.debug(r.shape)
return r, hidden