def generate_case_from_nntxt_str(nntxt_str, param_format, dataset_sample_num, batch_size=None): proto = proto_from_str(nntxt_str) with generate_csv_png(dataset_sample_num, get_input_size(proto)) as dataset_csv_file: # To test dataset, we create a randomly generated dataset. for ds in proto.dataset: ds.batch_size = batch_size if batch_size else ds.batch_size ds.uri = dataset_csv_file ds.cache_dir = os.path.join(os.path.dirname(dataset_csv_file), "data.cache") nntxt_io = io.StringIO() text_format.PrintMessage(proto, nntxt_io) nntxt_io.seek(0) version = io.StringIO() version.write('{}\n'.format(nnp_version())) version.seek(0) param = io.BytesIO() prepare_parameters(nntxt_str) nn.parameter.save_parameters(param, extension=param_format) with create_temp_with_dir(NNP_FILE) as temp_nnp_file_name: with get_file_handle_save(temp_nnp_file_name, ".nnp") as nnp: nnp.writestr('nnp_version.txt', version.read()) nnp.writestr('network.nntxt', nntxt_io.read()) nnp.writestr('parameter{}'.format(param_format), param.read()) yield temp_nnp_file_name
def _save_parameters(tmp_nnp_file, train_config, nntxt_str): base = os.path.dirname(tmp_nnp_file) base = os.path.join(base, 'results') version_filename = base + '_version.txt' with open(version_filename, 'w') as file: file.write('{}\n'.format(nnp_version())) # This is for testing, start=> nntxt_filename = base + '_network.nntxt' with open(nntxt_filename, "w") as file: file.write(nntxt_str) # <= End. param_filename = base + '_param.h5' nn.parameter.save_parameters(param_filename) opti_filenames = save_optimizer_states(base, '.h5', train_config) with zipfile.ZipFile(tmp_nnp_file, 'w') as nnp: nnp.write(version_filename, 'nnp_version.txt') nnp.write(nntxt_filename, "network.nntxt") nnp.write(param_filename, 'parameter.h5') for f in opti_filenames: nnp.write(f, f[len(base) + 1:]) os.unlink(version_filename) os.unlink(param_filename) for f in opti_filenames: os.unlink(f) logger.info("{} is saved.".format(tmp_nnp_file))
def _save_parameters(args, suffix, epoch, force=False): global _save_parameter_info if suffix not in _save_parameter_info: _save_parameter_info[suffix] = {} _save_parameter_info[suffix]['epoch'] = 0 _save_parameter_info[suffix]['time'] = 0 current_time = time.time() timediff = current_time - _save_parameter_info[suffix]['time'] epochdiff = epoch - _save_parameter_info[suffix]['epoch'] globname = os.path.join(args.outdir, 'results_{}_*.nnp'.format(suffix)) exists = glob.glob(globname) base = os.path.join(args.outdir, 'results_{}_{}'.format(suffix, epoch)) base_candidate = callback.result_base(base, suffix, args.outdir) if base_candidate is None: if suffix is None or suffix == 'best': base = os.path.join(args.outdir, 'results') else: base = base_candidate filename = base + '.nnp' if force or (not os.path.exists(filename) and (timediff > 180.0 or epochdiff > 10)): # Remove existing nnp before saving new file. for exist in exists: os.unlink(exist) version_filename = base + '_version.txt' with open(version_filename, 'w') as file: file.write('{}\n'.format(nnp_version())) param_filename = base + '_param.protobuf' save_parameters(param_filename) with zipfile.ZipFile(filename, 'w') as nnp: nnp.write(version_filename, 'nnp_version.txt') nnp.write(_save_parameter_info['config'], os.path.basename(_save_parameter_info['config'])) nnp.write(param_filename, 'parameter.protobuf') os.unlink(version_filename) os.unlink(param_filename) _save_parameter_info[suffix]['epoch'] = epoch _save_parameter_info[suffix]['time'] = current_time callback.save_train_snapshot()
def _save_parameters(args, suffix, epoch, force=False): global _save_parameter_info if suffix not in _save_parameter_info: _save_parameter_info[suffix] = {} _save_parameter_info[suffix]['epoch'] = 0 _save_parameter_info[suffix]['time'] = 0 current_time = time.time() timediff = current_time - _save_parameter_info[suffix]['time'] epochdiff = epoch - _save_parameter_info[suffix]['epoch'] globname = os.path.join(args.outdir, 'results_{}_*.nnp'.format(suffix)) exists = glob.glob(globname) base = os.path.join(args.outdir, 'results_{}_{}'.format(suffix, epoch)) if suffix == 'best': base = os.path.join(args.outdir, 'results') filename = base + '.nnp' if not os.path.exists(filename) and \ (force or timediff > 180.0 or epochdiff > 10): version_filename = base + '_version.txt' with open(version_filename, 'w') as file: file.write('{}\n'.format(nnp_version())) param_filename = base + '_param.protobuf' save_parameters(param_filename) with zipfile.ZipFile(filename, 'w') as nnp: nnp.write(version_filename, 'nnp_version.txt') nnp.write(_save_parameter_info['config'], os.path.basename(_save_parameter_info['config'])) nnp.write(param_filename, 'parameter.protobuf') os.unlink(version_filename) os.unlink(param_filename) for exist in exists: os.unlink(exist) _save_parameter_info[suffix]['epoch'] = epoch _save_parameter_info[suffix]['time'] = current_time
def _nnp_file_saver(ctx, filename, ext): logger.info("Saving {} as nnp".format(filename)) nntxt = io.StringIO() _nntxt_file_saver(ctx, nntxt, ".nntxt") version = io.StringIO() version.write('{}\n'.format(nnp_version())) version.seek(0) param = io.BytesIO() if ctx.parameters is None: nn.parameter.save_parameters(param, extension='.protobuf') else: nn.parameter.save_parameters( param, ctx.parameters, extension='.protobuf') with get_file_handle_save(filename, ext) as nnp: nnp.writestr('nnp_version.txt', version.read()) nnp.writestr('network.nntxt', nntxt.read()) nnp.writestr('parameter.protobuf', param.read())
def save(filename, contents, include_params=False, variable_batch_size=True): '''Save network definition, inference/training execution configurations etc. Args: filename (str): Filename to store information. The file extension is used to determine the saving file format. ``.nnp``: (Recommended) Creating a zip archive with nntxt (network definition etc.) and h5 (parameters). ``.nntxt``: Protobuf in text format. ``.protobuf``: Protobuf in binary format (unsafe in terms of backward compatibility). contents (dict): Information to store. include_params (bool): Includes parameter into single file. This is ignored when the extension of filename is nnp. variable_batch_size (bool): By ``True``, the first dimension of all variables is considered as batch size, and left as a placeholder (more specifically ``-1``). The placeholder dimension will be filled during/after loading. Example: The following example creates a two inputs and two outputs MLP, and save the network structure and the initialized parameters. .. code-block:: python import nnabla as nn import nnabla.functions as F import nnabla.parametric_functions as PF from nnabla.utils.save import save batch_size = 16 x0 = nn.Variable([batch_size, 100]) x1 = nn.Variable([batch_size, 100]) h1_0 = PF.affine(x0, 100, name='affine1_0') h1_1 = PF.affine(x1, 100, name='affine1_0') h1 = F.tanh(h1_0 + h1_1) h2 = F.tanh(PF.affine(h1, 50, name='affine2')) y0 = PF.affine(h2, 10, name='affiney_0') y1 = PF.affine(h2, 10, name='affiney_1') contents = { 'networks': [ {'name': 'net1', 'batch_size': batch_size, 'outputs': {'y0': y0, 'y1': y1}, 'names': {'x0': x0, 'x1': x1}}], 'executors': [ {'name': 'runtime', 'network': 'net1', 'data': ['x0', 'x1'], 'output': ['y0', 'y1']}]} save('net.nnp', contents) To get a trainable model, use following code instead. .. code-block:: python contents = { 'global_config': {'default_context': ctx}, 'training_config': {'max_epoch': args.max_epoch, 'iter_per_epoch': args_added.iter_per_epoch, 'save_best': True}, 'networks': [ {'name': 'training', 'batch_size': args.batch_size, 'outputs': {'loss': loss_t}, 'names': {'x': x, 'y': t, 'loss': loss_t}}, {'name': 'validation', 'batch_size': args.batch_size, 'outputs': {'loss': loss_v}, 'names': {'x': x, 'y': t, 'loss': loss_v}}], 'optimizers': [ {'name': 'optimizer', 'solver': solver, 'network': 'training', 'dataset': 'mnist_training', 'weight_decay': 0, 'lr_decay': 1, 'lr_decay_interval': 1, 'update_interval': 1}], 'datasets': [ {'name': 'mnist_training', 'uri': 'MNIST_TRAINING', 'cache_dir': args.cache_dir + '/mnist_training.cache/', 'variables': {'x': x, 'y': t}, 'shuffle': True, 'batch_size': args.batch_size, 'no_image_normalization': True}, {'name': 'mnist_validation', 'uri': 'MNIST_VALIDATION', 'cache_dir': args.cache_dir + '/mnist_test.cache/', 'variables': {'x': x, 'y': t}, 'shuffle': False, 'batch_size': args.batch_size, 'no_image_normalization': True }], 'monitors': [ {'name': 'training_loss', 'network': 'validation', 'dataset': 'mnist_training'}, {'name': 'validation_loss', 'network': 'validation', 'dataset': 'mnist_validation'}], } ''' _, ext = os.path.splitext(filename) if ext == '.nntxt' or ext == '.prototxt': logger.info("Saving {} as prototxt".format(filename)) proto = create_proto(contents, include_params, variable_batch_size) with open(filename, 'w') as file: text_format.PrintMessage(proto, file) elif ext == '.protobuf': logger.info("Saving {} as protobuf".format(filename)) proto = create_proto(contents, include_params, variable_batch_size) with open(filename, 'wb') as file: file.write(proto.SerializeToString()) elif ext == '.nnp': logger.info("Saving {} as nnp".format(filename)) try: tmpdir = tempfile.mkdtemp() save('{}/network.nntxt'.format(tmpdir), contents, include_params=False, variable_batch_size=variable_batch_size) with open('{}/nnp_version.txt'.format(tmpdir), 'w') as file: file.write('{}\n'.format(nnp_version())) save_parameters('{}/parameter.protobuf'.format(tmpdir)) with zipfile.ZipFile(filename, 'w') as nnp: nnp.write('{}/nnp_version.txt'.format(tmpdir), 'nnp_version.txt') nnp.write('{}/network.nntxt'.format(tmpdir), 'network.nntxt') nnp.write('{}/parameter.protobuf'.format(tmpdir), 'parameter.protobuf') finally: shutil.rmtree(tmpdir)
def _save_parameters(args, suffix, epoch, train_config, force=False): global _save_parameter_info if suffix not in _save_parameter_info: _save_parameter_info[suffix] = {} _save_parameter_info[suffix]['epoch'] = 0 _save_parameter_info[suffix]['time'] = 0 current_time = time.time() timediff = current_time - _save_parameter_info[suffix]['time'] epochdiff = epoch - _save_parameter_info[suffix]['epoch'] globname = os.path.join(args.outdir, 'results_{}_*.nnp'.format(suffix)) exists = glob.glob(globname) base = os.path.join(args.outdir, 'results_{}_{}'.format(suffix, epoch)) base_candidate = callback.result_base(base, suffix, args.outdir) if base_candidate is None: if suffix is None or suffix == 'best': base = os.path.join(args.outdir, 'results') else: base = base_candidate filename = base + '.nnp' if force or (not os.path.exists(filename) and (timediff > 180.0 or epochdiff > 10)): # Remove existing nnp before saving new file. for exist in exists: os.unlink(exist) version_filename = base + '_version.txt' with open(version_filename, 'w') as file: file.write('{}\n'.format(nnp_version())) param_filename = base + '_param.h5' save_parameters(param_filename) need_save_opti = train_config.optimizers and epoch % _OPTIMIZER_CHECKPOINT_INTERVAL == 0 if need_save_opti: opti_filenames = save_optimizer_states(base, '.h5', train_config) with zipfile.ZipFile(filename, 'w') as nnp: nnp.write(version_filename, 'nnp_version.txt') nnp.write(_save_parameter_info['config'], os.path.basename(_save_parameter_info['config'])) nnp.write(param_filename, 'parameter.h5') if need_save_opti: for f in opti_filenames: nnp.write(f, f[len(base) + 1:]) os.unlink(version_filename) os.unlink(param_filename) if need_save_opti: for f in opti_filenames: os.unlink(f) _save_parameter_info[suffix]['epoch'] = epoch _save_parameter_info[suffix]['time'] = current_time callback.save_train_snapshot()