def train_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
class TrainConfig:
pass
config = TrainConfig()
info = load.load(files)
logger.log(99, 'Train with contexts {}'.format(available_contexts))
config.global_config = info.global_config
config.training_config = info.training_config
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
# Training
max_iter = config.training_config.max_epoch * \
config.training_config.iter_per_epoch
if max_iter > 0:
data_iterators = {'optimizer': {}, 'monitor': {}}
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
for name, m in config.monitors.items():
m.data_iterator = stack.enter_context(
m.monitor.data_iterator())
train(args, config)
else:
# save parameters without training (0 epoch learning)
save_parameters(os.path.join(args.outdir, 'parameters.h5'))
logger.log(99, 'Training Completed.')
progress(None)
def profile_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
class TrainConfig:
pass
config = TrainConfig()
info = load.load(files)
config.global_config = info.global_config
config.training_config = info.training_config
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
ext_module = import_extension_module(
config.global_config.default_context.backend[0].split(':')[0])
def synchronize():
return ext_module.synchronize(
device_id=config.global_config.default_context.device_id)
result_array = [['time in ms']]
# Profile Optimizer
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(o.optimizer.data_iterator())
result_array = profile_optimizer(config, result_array, synchronize)
# Write profiling result
import csv
with open(args.outdir + os.sep + 'profile.csv', 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(result_array)
logger.log(99, 'Profile Completed.')
progress(None)
return True
def train_command(args):
logger.log(99, 'Train with contexts {}'.format(available_contexts))
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
class TrainConfig:
pass
config = TrainConfig()
info = load.load(files)
config.global_config = info.global_config
config.training_config = info.training_config
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
# Training
max_iter = config.training_config.max_epoch * \
config.training_config.iter_per_epoch
if max_iter > 0:
data_iterators = {'optimizer': {}, 'monitor': {}}
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
for name, m in config.monitors.items():
m.data_iterator = stack.enter_context(
m.monitor.data_iterator())
train(args, config)
else:
# save parameters without training (0 epoch learning)
save_parameters(os.path.join(
args.outdir, 'parameters.h5'))
logger.log(99, 'Training Completed.')
progress(None)
def profile_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
class TrainConfig:
pass
config = TrainConfig()
info = load.load(files)
config.global_config = info.global_config
config.training_config = info.training_config
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
result_array = [['time in ms']]
# Profile Optimizer
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
result_array = profile_optimizer(config, result_array)
# Write profiling result
import csv
with open(args.outdir + os.sep + 'profile.csv', 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(result_array)
logger.log(99, 'Profile Completed.')
progress(None)
def train_command(args):
callback.update_status(args)
if single_or_rankzero():
configure_progress(os.path.join(args.outdir, 'progress.txt'))
info = load.load([args.config],
prepare_data_iterator=None,
exclude_parameter=True)
# Check dataset uri is empty.
dataset_error = False
for dataset in info.datasets.values():
if dataset.uri.strip() == '':
dataset_error = True
if dataset_error:
logger.log(99, 'Fatal error. Dataset URI is empty.')
return False
class TrainConfig:
pass
config = TrainConfig()
config.timelimit = -1
if args.param:
load.load([args.param], parameter_only=True)
config.timelimit = callback.get_timelimit(args)
config.global_config = info.global_config
config.training_config = info.training_config
if single_or_rankzero():
logger.log(99, 'Train with contexts {}'.format(available_contexts))
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterators = []
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterators = []
config.monitors[name] = m
# Training
comm = current_communicator()
config.training_config.iter_per_epoch //= comm.size if comm else 1
max_iteration = config.training_config.max_epoch * \
config.training_config.iter_per_epoch
global _save_parameter_info
_save_parameter_info = {}
_, config_ext = os.path.splitext(args.config)
if config_ext == '.prototxt' or config_ext == '.nntxt':
_save_parameter_info['config'] = args.config
elif config_ext == '.nnp':
with zipfile.ZipFile(args.config, 'r') as nnp:
for name in nnp.namelist():
_, ext = os.path.splitext(name)
if ext == '.nntxt' or ext == '.prototxt':
nnp.extract(name, args.outdir)
_save_parameter_info['config'] = os.path.join(
args.outdir, name)
result = False
restart = False
if max_iteration > 0:
rng = np.random.RandomState(comm.rank if comm else 0)
with ExitStack() as stack:
# Create data_iterator instance only once for each dataset in optimizers
optimizer_data_iterators = {}
for name, o in config.optimizers.items():
for di in o.optimizer.data_iterators.values():
if di not in optimizer_data_iterators:
di_instance = stack.enter_context(di())
if comm and comm.size > 1:
di_instance = di_instance.slice(
rng, comm.size, comm.rank)
optimizer_data_iterators[di] = di_instance
else:
di_instance = optimizer_data_iterators[di]
o.data_iterators.append(di_instance)
# Create data_iterator instance only once for each dataset in monitors
monitor_data_iterators = {}
for name, m in config.monitors.items():
for di in m.monitor.data_iterators.values():
if di not in monitor_data_iterators:
di_instance = stack.enter_context(di())
if comm and comm.size > 1:
di_instance = di_instance.slice(
rng, comm.size, comm.rank)
monitor_data_iterators[di] = di_instance
else:
di_instance = monitor_data_iterators[di]
m.data_iterators.append(di_instance)
monitor_data_iterators.update(optimizer_data_iterators)
result, restart = _train(args, config)
else:
# save parameters without training (0 epoch learning)
logger.log(99, '0 epoch learning. (Just save parameter.)')
if single_or_rankzero():
_save_parameters(args, None, 0, config, True)
result = True
if single_or_rankzero() and not restart:
if result:
logger.log(99, 'Training Completed.')
callback.update_status('finished')
else:
logger.log(99, 'Training Incompleted.')
callback.update_status('failed')
if single_or_rankzero():
progress(None)
return True
def forward_command(args):
callback.update_status(args)
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
batch_size = args.batch_size
if batch_size < 1:
batch_size = None
class ForwardConfig:
pass
config = ForwardConfig
info = load.load(files, prepare_data_iterator=False, batch_size=batch_size)
config.global_config = info.global_config
config.executors = info.executors.values()
config.networks = []
for e in config.executors:
if e.network.name in info.networks.keys():
config.networks.append(info.networks[e.network.name])
else:
logger.critical('Network {} is not found.'.format(
config.executor.network.name))
return False
normalize = True
for d in info.datasets.values():
if d.uri == args.dataset or d.cache_dir == args.dataset:
normalize = d.normalize
for e in config.executors:
normalize = normalize and not e.no_image_normalization
orders = {}
# With CSV
if os.path.splitext(args.dataset)[1] == '.csv':
data_iterator = (lambda: data_iterator_csv_dataset(
uri=args.dataset,
batch_size=config.networks[0].batch_size,
shuffle=False,
normalize=normalize,
with_memory_cache=False,
with_file_cache=False))
# load dataset as csv
filereader = FileReader(args.dataset)
with filereader.open(textmode=True, encoding='utf-8-sig') as f:
rows = [row for row in csv.reader(f)]
row0 = rows.pop(0)
if args.replace_path:
root_path = os.path.dirname(args.dataset)
root_path = os.path.abspath(root_path.replace('/|\\', os.path.sep))
else:
root_path = '.'
rows = [row for row in rows if len(row)]
rows = list(
map(
lambda row: list(
map(
lambda i, x: x if row0[i][0] == '#' or is_float(
x) else compute_full_path(root_path, x),
range(len(row)), row)), rows))
for i in range(len(rows)):
orders[i] = i
# With Cache
elif os.path.splitext(args.dataset)[1] == '.cache':
data_iterator = (lambda: data_iterator_cache(uri=args.dataset,
batch_size=config.
networks[0].batch_size,
shuffle=False,
normalize=normalize))
# Get original CSV
original_csv = os.path.join(args.dataset, 'original.csv')
try:
# load dataset as csv
filereader = FileReader(original_csv)
with filereader.open(textmode=True, encoding='utf-8-sig') as f:
rows = [row for row in csv.reader(f)]
row0 = rows.pop(0)
root_path = '.'
rows = list(
map(
lambda row: list(
map(
lambda x: x if is_float(x) else compute_full_path(
root_path, x), row)), rows))
except:
print('Cannot open', original_csv)
pass
# Get original Data order.
order_csv = os.path.join(args.dataset, 'order.csv')
try:
filereader = FileReader(order_csv)
with filereader.open(textmode=True) as f:
for original, shuffled in [[int(x) for x in row]
for row in csv.reader(f)]:
orders[original] = shuffled
except:
print('Cannot open', order_csv)
for i in range(len(rows)):
orders[i] = i
else:
print('Unsupported extension "{}" in "{}".'.format(
os.path.splitext(args.dataset)[1], args.dataset))
callback.update_status(('data.max', len(rows)))
callback.update_status(('data.current', 0))
callback.update_status('processing', True)
result_csv_filename = os.path.join(args.outdir, args.outfile)
with open(result_csv_filename, 'w', encoding='utf-8') as f:
writer = csv.writer(f, lineterminator='\n')
with data_iterator() as di:
index = 0
while index < di.size:
data = di.next()
result, outputs = _forward(args, index, config, data,
di.variables)
if index == 0:
for name, dim in zip(result.names, result.dims):
if dim == 1:
if e.repeat_evaluation_type == "std":
name = "Uncertainty(Std)"
row0.append(name)
else:
for d in range(dim):
row0.append(name + '__' + str(d))
writer.writerow(row0)
for i, output in enumerate(outputs):
if index + i < len(rows):
import copy
row = copy.deepcopy(rows[orders[index + i]])
row.extend(output)
writer.writerow(row)
index += len(outputs)
callback.update_status(('data.current', min([index,
len(rows)])))
callback.update_forward_time()
callback.update_status()
logger.log(
99, 'data {} / {}'.format(min([index, len(rows)]),
len(rows)))
callback.process_evaluation_result(args.outdir, result_csv_filename)
logger.log(99, 'Forward Completed.')
progress(None)
callback.update_status(('output_result.csv_header', ','.join(row0)))
callback.update_status(('output_result.column_num', len(row0)))
callback.update_status(('output_result.data_num', len(rows)))
callback.update_status('finished')
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('input',
type=str,
nargs='+',
help='Source file or directory.')
parser.add_argument('output', type=str, help='Destination directory.')
parser.add_argument('-W',
'--width',
type=int,
default=320,
help='width of output image (default:320)')
parser.add_argument('-H',
'--height',
type=int,
default=320,
help='height of output image (default:320)')
parser.add_argument(
'-m',
'--mode',
default='trimming',
choices=['trimming', 'padding'],
help='shaping mode (trimming or padding) (default:trimming)')
parser.add_argument(
'-S',
'--shuffle',
choices=['True', 'False'],
help='shuffle mode if not specified, train:True, val:False.' +
' Otherwise specified value will be used for both.')
parser.add_argument('-N',
'--file-cache-size',
type=int,
default=100,
help='num of data in cache file (default:100)')
parser.add_argument('-C',
'--cache-type',
default='npy',
choices=['h5', 'npy'],
help='cache format (h5 or npy) (default:npy)')
parser.add_argument('--thinning',
type=int,
default=1,
help='Thinning rate')
args = parser.parse_args()
############################################################################
# Analyze tar
# If it consists only of members corresponding to regular expression
# 'n[0-9]{8}\.tar', it is judged as train data archive.
# If it consists only of members corresponding to regular expression
# 'ILSVRC2012_val_[0-9]{8}\.JPEG', it is judged as validation data archive.
archives = {'train': None, 'val': None}
for inputarg in args.input:
print('Checking input file [{}]'.format(inputarg))
archive = tarfile.open(inputarg)
is_train = False
is_val = False
names = []
for name in archive.getnames():
if re.match(r'n[0-9]{8}\.tar', name):
if is_val:
print('Train data {} includes in validation tar'.format(
name))
exit(-1)
is_train = True
elif re.match(r'ILSVRC2012_val_[0-9]{8}\.JPEG', name):
if is_train:
print('Validation data {} includes in train tar'.format(
name))
exit(-1)
is_val = True
else:
print('Invalid member {} includes in tar file'.format(name))
exit(-1)
names.append(name)
if is_train:
if archives['train'] is None:
archives['train'] = (archive, names)
else:
print('Please specify only 1 training tar archive.')
exit(-1)
if is_val:
if archives['val'] is None:
archives['val'] = (archive, names)
else:
print('Please specify only 1 validation tar archive.')
exit(-1)
# Read label of validation data, (Use ascending label of wordnet_id)
validation_ground_truth = []
g_file = VALIDATION_DATA_LABEL
with open(g_file, 'r') as f:
for l in f.readlines():
validation_ground_truth.append(int(l.rstrip()))
############################################################################
# Prepare logging
tmpdir = tempfile.mkdtemp()
logfilename = os.path.join(tmpdir, 'nnabla.log')
# Temporarily chdir to tmpdir just before importing nnabla to reflect nnabla.conf.
cwd = os.getcwd()
os.chdir(tmpdir)
with open('nnabla.conf', 'w') as f:
f.write('[LOG]\n')
f.write('log_file_name = {}\n'.format(logfilename))
f.write('log_file_format = %(funcName)s : %(message)s\n')
f.write('log_console_level = CRITICAL\n')
from nnabla.config import nnabla_config
os.chdir(cwd)
############################################################################
# Data iterator setting
nnabla_config.set('DATA_ITERATOR', 'cache_file_format',
'.' + args.cache_type)
nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size',
str(args.file_cache_size))
nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_num_of_threads',
'1')
if not os.path.isdir(args.output):
os.makedirs(args.output)
############################################################################
# Prepare status monitor
from nnabla.utils.progress import configure_progress
configure_progress(None, _progress)
############################################################################
# Converter
try:
if archives['train'] is not None:
from nnabla.logger import logger
logger.info('StartCreatingCache')
archive, names = archives['train']
output = os.path.join(args.output, 'train')
if not os.path.isdir(output):
os.makedirs(output)
_create_train_cache(archive, output, names, args)
if archives['val'] is not None:
from nnabla.logger import logger
logger.info('StartCreatingCache')
archive, names = archives['val']
output = os.path.join(args.output, 'val')
if not os.path.isdir(output):
os.makedirs(output)
_create_validation_cache(archive, output, names,
validation_ground_truth, args)
except KeyboardInterrupt:
shutil.rmtree(tmpdir, ignore_errors=True)
# Even if CTRL-C is pressed, it does not stop if there is a running
# thread, so it sending a signal to itself.
os.kill(os.getpid(), 9)
############################################################################
# Finish
_finish = True
shutil.rmtree(tmpdir, ignore_errors=True)
def train_command(args):
if single_or_rankzero():
configure_progress(os.path.join(args.outdir, 'progress.txt'))
info = load.load([args.config], exclude_parameter=True)
# Check dataset uri is empty.
dataset_error = False
for dataset in info.datasets.values():
if dataset.uri.strip() == '':
dataset_error = True
if dataset_error:
logger.log(99, 'Fatal error. Dataset URI is empty.')
return False
class TrainConfig:
pass
config = TrainConfig()
config.timelimit = -1
if args.param:
load.load([args.param], parameter_only=True)
config.global_config = info.global_config
config.training_config = info.training_config
if single_or_rankzero():
logger.log(99, 'Train with contexts {}'.format(available_contexts))
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
# Training
comm = current_communicator()
config.training_config.iter_per_epoch //= comm.size if comm else 1
max_iteration = config.training_config.max_epoch * \
config.training_config.iter_per_epoch
global _save_parameter_info
_save_parameter_info = {}
_, config_ext = os.path.splitext(args.config)
if config_ext == '.prototxt' or config_ext == '.nntxt':
_save_parameter_info['config'] = args.config
elif config_ext == '.nnp':
with zipfile.ZipFile(args.config, 'r') as nnp:
for name in nnp.namelist():
_, ext = os.path.splitext(name)
if ext == '.nntxt' or ext == '.prototxt':
nnp.extract(name, args.outdir)
_save_parameter_info['config'] = os.path.join(
args.outdir, name)
result = False
if max_iteration > 0:
data_iterators = {'optimizer': {}, 'monitor': {}}
rng = np.random.RandomState(comm.rank if comm else 0)
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
if comm and comm.size > 1:
o.data_iterator = o.data_iterator.slice(
rng, comm.size, comm.rank)
for name, m in config.monitors.items():
m.data_iterator = stack.enter_context(
m.monitor.data_iterator())
if comm and comm.size > 1:
m.data_iterator = m.data_iterator.slice(
rng, comm.size, comm.rank)
result = _train(args, config)
else:
# save parameters without training (0 epoch learning)
logger.log(99, '0 epoch learning. (Just save parameter.)')
if single_or_rankzero():
_save_parameters(args, 'current', 0, True)
result = True
if single_or_rankzero():
if result:
logger.log(99, 'Training Completed.')
else:
logger.log(99, 'Training Incompleted.')
if single_or_rankzero():
progress(None)
return True
def forward_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
batch_size = args.batch_size
if batch_size < 1:
batch_size = None
class ForwardConfig:
pass
config = ForwardConfig
info = load.load(files, prepare_data_iterator=False, batch_size=batch_size)
config.global_config = info.global_config
config.executors = info.executors.values()
config.networks = []
for e in config.executors:
if e.network.name in info.networks.keys():
config.networks.append(info.networks[e.network.name])
else:
logger.critical('Network {} is not found.'.format(
config.executor.network.name))
return False
normalize = True
for d in info.datasets.values():
if d.uri == args.dataset:
normalize = d.normalize
for e in config.executors:
normalize = normalize and not e.no_image_normalization
data_iterator = (lambda: data_iterator_csv_dataset(uri=args.dataset,
batch_size=config.
networks[0].batch_size,
shuffle=False,
normalize=normalize,
with_memory_cache=False,
with_file_cache=False))
# load dataset as csv
filereader = FileReader(args.dataset)
with filereader.open(textmode=True) as f:
rows = [row for row in csv.reader(f)]
row0 = rows.pop(0)
root_path = os.path.dirname(args.dataset)
root_path = os.path.abspath(root_path.replace('/|\\', os.path.sep))
rows = list(
map(
lambda row: list(
map(
lambda x: x
if is_float(x) else compute_full_path(root_path, x), row)),
rows))
with open(os.path.join(args.outdir, 'output_result.csv'), 'w') as f:
writer = csv.writer(f, lineterminator='\n')
with data_iterator() as di:
index = 0
while index < di.size:
data = di.next()
result, outputs = _forward(args, index, config, data,
di.variables)
if index == 0:
for name, dim in zip(result.names, result.dims):
if dim == 1:
row0.append(name)
else:
for d in range(dim):
row0.append(name + '__' + str(d))
writer.writerow(row0)
for i, output in enumerate(outputs):
if index + i < len(rows):
import copy
row = copy.deepcopy(rows[index + i])
row.extend(output)
writer.writerow(row)
index += len(outputs)
logger.log(
99, 'data {} / {}'.format(min([index, len(rows)]),
len(rows)))
logger.log(99, 'Forward Completed.')
progress(None)
return True
def compare_with_cpu_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
class TrainConfig:
pass
class OptConfig:
pass
class MonConfig:
pass
# Load config with current context
files = []
files.append(args.config)
with nn.parameter_scope('current'):
info = load.load(files)
parameters = get_parameters(grad_only=False)
config = TrainConfig()
config.global_config = info.global_config
config.training_config = info.training_config
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
# Load config with cpu context
files = []
files.append(args.config2)
with nn.parameter_scope('cpu'):
info_cpu = load.load(files)
cpu_parameters = get_parameters(grad_only=False)
config_cpu = TrainConfig()
config_cpu.global_config = info_cpu.global_config
config_cpu.training_config = info_cpu.training_config
config_cpu.optimizers = OrderedDict()
for name, opt in info_cpu.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config_cpu.optimizers[name] = o
config_cpu.monitors = OrderedDict()
for name, mon in info_cpu.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config_cpu.monitors[name] = m
result_array = [['1-Correl']]
# Profile Optimizer
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
for name, o in config_cpu.optimizers.items():
o.data_iterator = stack.enter_context(
o.optimizer.data_iterator())
result_array = compare_optimizer(
config, parameters, config_cpu, cpu_parameters, result_array)
# Write profiling result
import csv
with open(args.outdir + os.sep + 'compare_with_cpu.csv', 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(result_array)
logger.log(99, 'Compare with CPU Completed.')
progress(None)
def profile_command(args):
callback.update_status(args)
configure_progress(os.path.join(args.outdir, 'progress.txt'))
class TrainConfig:
pass
config = TrainConfig()
info = load.load(args.config)
config.global_config = info.global_config
config.training_config = info.training_config
class OptConfig:
pass
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterators = []
config.optimizers[name] = o
class MonConfig:
pass
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterators = []
config.monitors[name] = m
ext_module = import_extension_module(
config.global_config.default_context.backend[0].split(':')[0])
def synchronize():
return ext_module.synchronize(
device_id=config.global_config.default_context.device_id)
result_array = [['time in ms']]
callback.update_status('processing', True)
# Profile Optimizer
with ExitStack() as stack:
# Create data_iterator instance only once for each dataset in optimizers
optimizer_data_iterators = {}
for name, o in config.optimizers.items():
for di in o.optimizer.data_iterators.values():
if di not in optimizer_data_iterators:
di_instance = stack.enter_context(di())
optimizer_data_iterators[di] = di_instance
else:
di_instance = optimizer_data_iterators[di]
o.data_iterators.append(di_instance)
result_array = profile_optimizer(config, result_array, synchronize)
# Write profiling result
import csv
with open(args.outdir + os.sep + 'profile.csv', 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(result_array)
logger.log(99, 'Profile Completed.')
progress(None)
callback.update_status('finished')
return True
def compare_with_cpu_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
class TrainConfig:
pass
class OptConfig:
pass
class MonConfig:
pass
# Load config with current context
files = []
files.append(args.config)
with nn.parameter_scope('current'):
info = load.load(files)
parameters = get_parameters(grad_only=False)
config = TrainConfig()
config.global_config = info.global_config
config.training_config = info.training_config
config.optimizers = OrderedDict()
for name, opt in info.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config.optimizers[name] = o
config.monitors = OrderedDict()
for name, mon in info.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config.monitors[name] = m
# Load config with cpu context
files = []
files.append(args.config2)
with nn.parameter_scope('cpu'):
info_cpu = load.load(files)
cpu_parameters = get_parameters(grad_only=False)
config_cpu = TrainConfig()
config_cpu.global_config = info_cpu.global_config
config_cpu.training_config = info_cpu.training_config
config_cpu.optimizers = OrderedDict()
for name, opt in info_cpu.optimizers.items():
o = OptConfig()
o.optimizer = opt
o.data_iterator = None
config_cpu.optimizers[name] = o
config_cpu.monitors = OrderedDict()
for name, mon in info_cpu.monitors.items():
m = MonConfig()
m.monitor = mon
m.data_iterator = None
config_cpu.monitors[name] = m
result_array = [['1-Correl']]
# Profile Optimizer
with ExitStack() as stack:
for name, o in config.optimizers.items():
o.data_iterator = stack.enter_context(o.optimizer.data_iterator())
for name, o in config_cpu.optimizers.items():
o.data_iterator = stack.enter_context(o.optimizer.data_iterator())
result_array = compare_optimizer(config, parameters, config_cpu,
cpu_parameters, result_array)
# Write profiling result
import csv
with open(args.outdir + os.sep + 'compare_with_cpu.csv', 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(result_array)
logger.log(99, 'Compare with CPU Completed.')
progress(None)
return True
def forward_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
class ForwardConfig:
pass
config = ForwardConfig
info = load.load(files, prepare_data_iterator=False)
config.global_config = info.global_config
config.executors = info.executors.values()
config.networks = []
for e in config.executors:
if e.network.name in info.networks.keys():
config.networks.append(info.networks[e.network.name])
else:
logger.critical('Network {} does not found.'.format(
config.executor.network.name))
return
normalize = True
for d in info.datasets.values():
if d.uri == args.dataset:
normalize = d.normalize
data_iterator = (lambda: data_iterator_csv_dataset(
args.dataset, config.networks[0].batch_size, False, normalize=normalize))
# load dataset as csv
with open(args.dataset, 'rt') as f:
rows = [row for row in csv.reader(f)]
row0 = rows.pop(0)
root_path = os.path.dirname(args.dataset)
root_path = os.path.abspath(root_path.replace('/|\\', os.path.sep))
rows = list(map(lambda row: list(map(lambda x: x if is_float(
x) else compute_full_path(root_path, x), row)), rows))
with data_iterator() as di:
index = 0
while index < di.size:
data = di.next()
result, outputs = forward(args, index, config, data, di.variables)
if index == 0:
for name, dim in zip(result.names, result.dims):
if dim == 1:
row0.append(name)
else:
for d in range(dim):
row0.append(name + '__' + str(d))
for i, output in enumerate(outputs):
if index + i < len(rows):
rows[index + i].extend(output)
index += len(outputs)
logger.log(
99, 'data {} / {}'.format(min([index, len(rows)]), len(rows)))
with open(os.path.join(args.outdir, 'output_result.csv'), 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerow(row0)
writer.writerows(rows)
logger.log(99, 'Forward Completed.')
progress(None)
def forward_command(args):
configure_progress(os.path.join(args.outdir, 'progress.txt'))
files = []
files.append(args.config)
if args.param:
files.append(args.param)
class ForwardConfig:
pass
config = ForwardConfig
info = load.load(files, prepare_data_iterator=False)
config.global_config = info.global_config
config.executors = info.executors.values()
config.networks = []
for e in config.executors:
if e.network.name in info.networks.keys():
config.networks.append(info.networks[e.network.name])
else:
logger.critical('Network {} does not found.'.format(
config.executor.network.name))
return
normalize = True
for d in info.datasets.values():
if d.uri == args.dataset:
normalize = d.normalize
data_iterator = (lambda: data_iterator_csv_dataset(
args.dataset, config.networks[0].batch_size, False, padding=True, normalize=normalize))
# load dataset as csv
with open(args.dataset, 'rt') as f:
rows = [row for row in csv.reader(f)]
row0 = rows.pop(0)
root_path = os.path.dirname(args.dataset)
root_path = os.path.abspath(root_path.replace('/|\\', os.path.sep))
rows = map(lambda row: map(lambda x: x if is_float(
x) else compute_full_path(root_path, x), row), rows)
with data_iterator() as di:
index = 0
while index < di.size:
data = di.next()
result, outputs = forward(args, index, config, data, di.variables)
if index == 0:
for name, dim in zip(result.names, result.dims):
if dim == 1:
row0.append(name)
else:
for d in range(dim):
row0.append(name + '__' + str(d))
for i, output in enumerate(outputs):
if index + i < len(rows):
rows[index + i].extend(output)
index += len(outputs)
logger.log(
99, 'data {} / {}'.format(min([index, len(rows)]), len(rows)))
with open(os.path.join(args.outdir, 'output_result.csv'), 'w') as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerow(row0)
writer.writerows(rows)
logger.log(99, 'Forward Completed.')
progress(None)
