label_mean_args = { "gpu_chunk": 32768, "lst_file": file_label_mean, "file_format": file_format, "separate_lines": True, "has_labels": False } test_sets = DataReadStream(test_data_args, feat_dim) label_mean_sets = DataReadStream(label_mean_args, label_dim) return (init_states, test_sets, label_mean_sets) if __name__ == '__main__': args = parse_args() args.config.write(sys.stderr) decoding_method = args.config.get('train', 'method') contexts = parse_contexts(args) init_states, test_sets, label_mean_sets = prepare_data(args) state_names = [x[0] for x in init_states] batch_size = args.config.getint('train', 'batch_size') num_hidden = args.config.getint('arch', 'num_hidden') num_hidden_proj = args.config.getint('arch', 'num_hidden_proj') num_lstm_layer = args.config.getint('arch', 'num_lstm_layer') feat_dim = args.config.getint('data', 'xdim') label_dim = args.config.getint('data', 'ydim') out_file = args.config.get('data', 'out_file')
data_names=data_names, label_names=label_names, load_optimizer_states=True) else: model_loaded = mx.module.Module.load(prefix=model_path, epoch=model_num_epoch, context=contexts, data_names=data_names, label_names=label_names, load_optimizer_states=False) return model_loaded, model_num_epoch if __name__ == '__main__': if len(sys.argv) <= 1: raise Exception('cfg file path must be provided. ex)python main.py --configfile examplecfg.cfg') mx.random.seed(hash(datetime.now())) # set parameters from cfg file args = parse_args(sys.argv[1]) log_filename = args.config.get('common', 'log_filename') log = LogUtil(filename=log_filename).getlogger() # set parameters from data section(common) mode = args.config.get('common', 'mode') if mode not in ['train', 'predict', 'load']: raise Exception( 'Define mode in the cfg file first. train or predict or load can be the candidate for the mode.') # get meta file where character to number conversions are defined language = args.config.get('data', 'language') labelUtil = LabelUtil.getInstance() if language == "en": labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv")
reset_optimizer() module.set_params(*last_params) else: last_params = module.get_params() last_acc = curr_acc n_epoch += 1 # save checkpoints mx.model.save_checkpoint(get_checkpoint_path(args), n_epoch, module.symbol, *last_params) if n_epoch == num_epoch: break if __name__ == '__main__': args = parse_args() args.config.write(sys.stdout) training_method = args.config.get('train', 'method') contexts = parse_contexts(args) init_states, train_sets, dev_sets = prepare_data(args) state_names = [x[0] for x in init_states] batch_size = args.config.getint('train', 'batch_size') num_hidden = args.config.getint('arch', 'num_hidden') num_hidden_proj = args.config.getint('arch', 'num_hidden_proj') num_lstm_layer = args.config.getint('arch', 'num_lstm_layer') feat_dim = args.config.getint('data', 'xdim') label_dim = args.config.getint('data', 'ydim')
import config_util args = config_util.parse_args() config = config_util.load_config(args.config) print(config)
#!/usr/bin/python import os, sys import codecs import numpy as np import mxnet as mx from mxnet import gluon, nd from config_util import parse_args, get_checkpoint_path, parse_contexts from model import Generator, Discriminator from data_iter import SentenceIter import ttspacker packer = ttspacker.ttspacker() if __name__ == '__main__': args = parse_args('default.cfg') source_scp = args.config.get('test', 'test_source_scp') target_scp = args.config.get('test', 'test_target_scp') feat_dim = args.config.getint('data', 'feat_dim') segment_length = args.config.getint('train', 'segment_length') num_iteration = args.config.getint('train', 'num_iteration') G_learning_rate = args.config.getfloat('train', 'G_learning_rate') D_learning_rate = args.config.getfloat('train', 'D_learning_rate') momentum = args.config.getfloat('train', 'momentum') source_speaker = args.config.get('data', 'source_speaker') target_speaker = args.config.get('data', 'target_speaker') lambda_cyc = args.config.getfloat('train', 'lambda_cyc') lambda_id = args.config.getfloat('train', 'lambda_id') input_gv = args.config.get('data', 'source_gv') output_gv = args.config.get('data', 'target_gv') G_A_check_iter = args.config.getint('test', 'G_A_check_iter') G_B_check_iter = args.config.getint('test', 'G_B_check_iter')
def __init__(self): if len(sys.argv) <= 1: raise Exception('cfg file path must be provided. ' + 'ex)python main.py --configfile examplecfg.cfg') self.args = parse_args(sys.argv[1]) # set parameters from cfg file # give random seed self.random_seed = self.args.config.getint('common', 'random_seed') self.mx_random_seed = self.args.config.getint('common', 'mx_random_seed') # random seed for shuffling data list if self.random_seed != -1: np.random.seed(self.random_seed) # set mx.random.seed to give seed for parameter initialization if self.mx_random_seed != -1: mx.random.seed(self.mx_random_seed) else: mx.random.seed(hash(datetime.now())) # set log file name self.log_filename = self.args.config.get('common', 'log_filename') self.log = LogUtil(filename=self.log_filename).getlogger() # set parameters from data section(common) self.mode = self.args.config.get('common', 'mode') # get meta file where character to number conversions are defined self.contexts = parse_contexts(self.args) self.num_gpu = len(self.contexts) self.batch_size = self.args.config.getint('common', 'batch_size') # check the number of gpus is positive divisor of the batch size for data parallel self.is_batchnorm = self.args.config.getboolean('arch', 'is_batchnorm') self.is_bucketing = self.args.config.getboolean('arch', 'is_bucketing') # log current config self.config_logger = ConfigLogger(self.log) self.config_logger(self.args.config) default_bucket_key = 1600 self.args.config.set('arch', 'max_t_count', str(default_bucket_key)) self.args.config.set('arch', 'max_label_length', str(100)) self.labelUtil = LabelUtil() is_bi_graphemes = self.args.config.getboolean('common', 'is_bi_graphemes') load_labelutil(self.labelUtil, is_bi_graphemes, language="zh") self.args.config.set('arch', 'n_classes', str(self.labelUtil.get_count())) self.max_t_count = self.args.config.getint('arch', 'max_t_count') # self.load_optimizer_states = self.args.config.getboolean('load', 'load_optimizer_states') # load model self.model_loaded, self.model_num_epoch, self.model_path = load_model( self.args) self.model = STTBucketingModule(sym_gen=self.model_loaded, default_bucket_key=default_bucket_key, context=self.contexts) from importlib import import_module prepare_data_template = import_module( self.args.config.get('arch', 'arch_file')) init_states = prepare_data_template.prepare_data(self.args) width = self.args.config.getint('data', 'width') height = self.args.config.getint('data', 'height') self.model.bind(data_shapes=[ ('data', (self.batch_size, default_bucket_key, width * height)) ] + init_states, label_shapes=[ ('label', (self.batch_size, self.args.config.getint('arch', 'max_label_length'))) ], for_training=True) _, self.arg_params, self.aux_params = mx.model.load_checkpoint( self.model_path, self.model_num_epoch) self.model.set_params(self.arg_params, self.aux_params, allow_extra=True, allow_missing=True) try: from swig_wrapper import Scorer vocab_list = [ chars.encode("utf-8") for chars in self.labelUtil.byList ] self.log.info("vacab_list len is %d" % len(vocab_list)) _ext_scorer = Scorer(0.26, 0.1, self.args.config.get('common', 'kenlm'), vocab_list) lm_char_based = _ext_scorer.is_character_based() lm_max_order = _ext_scorer.get_max_order() lm_dict_size = _ext_scorer.get_dict_size() self.log.info("language model: " "is_character_based = %d," % lm_char_based + " max_order = %d," % lm_max_order + " dict_size = %d" % lm_dict_size) self.scorer = _ext_scorer # self.eval_metric = EvalSTTMetric(batch_size=self.batch_size, num_gpu=self.num_gpu, is_logging=True, # scorer=_ext_scorer) except ImportError: import kenlm km = kenlm.Model(self.args.config.get('common', 'kenlm')) self.scorer = km.score
def __init__(self): if len(sys.argv) <= 1: raise Exception('cfg file path must be provided. ' + 'ex)python main.py --configfile examplecfg.cfg') self.args = parse_args(sys.argv[1]) # set parameters from cfg file # give random seed self.random_seed = self.args.config.getint('common', 'random_seed') self.mx_random_seed = self.args.config.getint('common', 'mx_random_seed') # random seed for shuffling data list if self.random_seed != -1: np.random.seed(self.random_seed) # set mx.random.seed to give seed for parameter initialization if self.mx_random_seed != -1: mx.random.seed(self.mx_random_seed) else: mx.random.seed(hash(datetime.now())) # set log file name self.log_filename = self.args.config.get('common', 'log_filename') self.log = LogUtil(filename=self.log_filename).getlogger() # set parameters from data section(common) self.mode = self.args.config.get('common', 'mode') save_dir = 'checkpoints' model_name = self.args.config.get('common', 'prefix') max_freq = self.args.config.getint('data', 'max_freq') self.datagen = DataGenerator(save_dir=save_dir, model_name=model_name, max_freq=max_freq) self.datagen.get_meta_from_file( np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) self.buckets = json.loads(self.args.config.get('arch', 'buckets')) # get meta file where character to number conversions are defined self.contexts = parse_contexts(self.args) self.num_gpu = len(self.contexts) self.batch_size = self.args.config.getint('common', 'batch_size') # check the number of gpus is positive divisor of the batch size for data parallel self.is_batchnorm = self.args.config.getboolean('arch', 'is_batchnorm') self.is_bucketing = self.args.config.getboolean('arch', 'is_bucketing') # log current config self.config_logger = ConfigLogger(self.log) self.config_logger(self.args.config) default_bucket_key = 1600 self.args.config.set('arch', 'max_t_count', str(default_bucket_key)) self.args.config.set('arch', 'max_label_length', str(95)) self.labelUtil = LabelUtil() is_bi_graphemes = self.args.config.getboolean('common', 'is_bi_graphemes') load_labelutil(self.labelUtil, is_bi_graphemes, language="zh") self.args.config.set('arch', 'n_classes', str(self.labelUtil.get_count())) self.max_t_count = self.args.config.getint('arch', 'max_t_count') # self.load_optimizer_states = self.args.config.getboolean('load', 'load_optimizer_states') # load model self.model_loaded, self.model_num_epoch, self.model_path = load_model( self.args) # self.model = STTBucketingModule( # sym_gen=self.model_loaded, # default_bucket_key=default_bucket_key, # context=self.contexts # ) from importlib import import_module prepare_data_template = import_module( self.args.config.get('arch', 'arch_file')) init_states = prepare_data_template.prepare_data(self.args) width = self.args.config.getint('data', 'width') height = self.args.config.getint('data', 'height') for bucket in self.buckets: net, init_state_names, ll = self.model_loaded(bucket) net.save('checkpoints/%s-symbol.json' % bucket) input_shapes = dict([('data', (self.batch_size, default_bucket_key, width * height))] + init_states + [('label', (1, 18))]) # self.executor = net.simple_bind(ctx=mx.cpu(), **input_shapes) # self.model.bind(data_shapes=[('data', (self.batch_size, default_bucket_key, width * height))] + init_states, # label_shapes=[ # ('label', (self.batch_size, self.args.config.getint('arch', 'max_label_length')))], # for_training=True) symbol, self.arg_params, self.aux_params = mx.model.load_checkpoint( self.model_path, self.model_num_epoch) all_layers = symbol.get_internals() concat = all_layers['concat36457_output'] sm = mx.sym.SoftmaxOutput(data=concat, name='softmax') self.executor = sm.simple_bind(ctx=mx.cpu(), **input_shapes) # self.model.set_params(self.arg_params, self.aux_params, allow_extra=True, allow_missing=True) for key in self.executor.arg_dict.keys(): if key in self.arg_params: self.arg_params[key].copyto(self.executor.arg_dict[key]) init_state_names.remove('data') init_state_names.sort() self.states_dict = dict( zip(init_state_names, self.executor.outputs[1:])) self.input_arr = mx.nd.zeros( (self.batch_size, default_bucket_key, width * height)) try: from swig_wrapper import Scorer vocab_list = [ chars.encode("utf-8") for chars in self.labelUtil.byList ] self.log.info("vacab_list len is %d" % len(vocab_list)) _ext_scorer = Scorer(0.26, 0.1, self.args.config.get('common', 'kenlm'), vocab_list) lm_char_based = _ext_scorer.is_character_based() lm_max_order = _ext_scorer.get_max_order() lm_dict_size = _ext_scorer.get_dict_size() self.log.info("language model: " "is_character_based = %d," % lm_char_based + " max_order = %d," % lm_max_order + " dict_size = %d" % lm_dict_size) self.eval_metric = EvalSTTMetric(batch_size=self.batch_size, num_gpu=self.num_gpu, is_logging=True, scorer=_ext_scorer) except ImportError: import kenlm km = kenlm.Model(self.args.config.get('common', 'kenlm')) self.eval_metric = EvalSTTMetric(batch_size=self.batch_size, num_gpu=self.num_gpu, is_logging=True, scorer=km.score)