def test_mobilenetv2_quant(): set_seed(1) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") config = config_ascend_quant print("training configure: {}".format(config)) epoch_size = config.epoch_size # define network network = mobilenetV2(num_classes=config.num_classes) # define loss if config.label_smooth > 0: loss = CrossEntropyWithLabelSmooth( smooth_factor=config.label_smooth, num_classes=config.num_classes) else: loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') # define dataset dataset = create_dataset(dataset_path=dataset_path, config=config, repeat_num=1, batch_size=config.batch_size) step_size = dataset.get_dataset_size() # convert fusion network to quantization aware network quantizer = QuantizationAwareTraining(bn_fold=True, per_channel=[True, False], symmetric=[True, False]) network = quantizer.quantize(network) # get learning rate lr = Tensor(get_lr(global_step=config.start_epoch * step_size, lr_init=0, lr_end=0, lr_max=config.lr, warmup_epochs=config.warmup_epochs, total_epochs=epoch_size + config.start_epoch, steps_per_epoch=step_size)) # define optimization opt = nn.Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), lr, config.momentum, config.weight_decay) # define model model = Model(network, loss_fn=loss, optimizer=opt) print("============== Starting Training ==============") monitor = Monitor(lr_init=lr.asnumpy(), step_threshold=config.step_threshold) callback = [monitor] model.train(epoch_size, dataset, callbacks=callback, dataset_sink_mode=False) print("============== End Training ==============") export_time_used = 650 train_time = monitor.step_mseconds print('train_time_used:{}'.format(train_time)) assert train_time < export_time_used expect_avg_step_loss = 2.32 avg_step_loss = np.mean(np.array(monitor.losses)) print("average step loss:{}".format(avg_step_loss)) assert avg_step_loss < expect_avg_step_loss
def test_resnet50_quant(): set_seed(1) context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") config = config_quant print("training configure: {}".format(config)) epoch_size = config.epoch_size # define network net = resnet50_quant(class_num=config.class_num) net.set_train(True) # define loss if not config.use_label_smooth: config.label_smooth_factor = 0.0 loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) #loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) # define dataset dataset = create_dataset(dataset_path=dataset_path, config=config, repeat_num=1, batch_size=config.batch_size) step_size = dataset.get_dataset_size() # convert fusion network to quantization aware network net = quant.convert_quant_network(net, bn_fold=True, per_channel=[True, False], symmetric=[True, False]) # get learning rate lr = Tensor( get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine')) # define optimization opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, config.weight_decay, config.loss_scale) # define model #model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) model = Model(net, loss_fn=loss, optimizer=opt) print("============== Starting Training ==============") monitor = Monitor(lr_init=lr.asnumpy(), step_threshold=config.step_threshold) callbacks = [monitor] model.train(epoch_size, dataset, callbacks=callbacks, dataset_sink_mode=False) print("============== End Training ==============") expect_avg_step_loss = 2.40 avg_step_loss = np.mean(np.array(monitor.losses)) print("average step loss:{}".format(avg_step_loss)) assert avg_step_loss < expect_avg_step_loss
try: email_msgs_study = [] (_, _, monitor_files) = next(walk(monitor_path)) mlog.info( 'The following monitor configs are to be processed (count = {}) in the monitor folder {}: {}' .format(len(monitor_files), monitor_path, monitor_files)) for mnt_config in monitor_files: mlog.info('-------------------------------------------') mlog.info('Start processing monitor config file: "{}".'.format( mnt_config)) try: mnt = Monitor(Path(monitor_path) / mnt_config, mlog) mnt.start_monitor() except Exception as ex: # report unexpected error to log file _str = 'Unexpected Error "{}" occurred during processing monitor config file: {}\n{} ' \ .format(ex, mnt_config, traceback.format_exc()) mlog.error(_str) mlog.info('Finish processing monitor config file: "{}".'.format( mnt_config)) # create a dictionary to feed into template for preparing an email body template_feeder = { 'source_file_path': '{}/{}'.format(str(mnt.mtr_source_dir), str(mnt.mtr_source_file)), 'source_identified': str(mnt.mtr_source_path),
def train(self, train_x, train_y, epoch=100, batch_size=128, validation_data=None, valid_freq=100, patience=10, monitor=False): # Define the symbolic train model batch_index = T.iscalar('batch_index') x = T.matrix('x', dtype=train_x.get_value(borrow=True).dtype) y = T.vector('y', dtype=train_y.get_value(borrow=True).dtype) loss = self.loss(x, y) updates = self.optimizer.get_updates(loss, self.params) sample_num = train_x.get_value(borrow=True).shape[0] train_batch_num = int(np.ceil(sample_num / batch_size)) train_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: train_x[batch_index * batch_size: (batch_index + 1) * batch_size], y: train_y[batch_index * batch_size: (batch_index + 1) * batch_size] } ) train_acc_fn = theano.function([], self.score(train_x, train_y)) # Initialization for validation if validation_data is not None: valid = True (valid_x, valid_y) = validation_data valid_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: valid_x[batch_index * batch_size: (batch_index + 1) * batch_size], y: valid_y[batch_index * batch_size: (batch_index + 1) * batch_size] } ) valid_acc_fn = theano.function([], self.score(valid_x, valid_y)) valid_batch_num = int(np.ceil(valid_x.get_value(borrow=True).shape[0] / batch_size)) valid_losses = [] valid_acces = [] best_valid_acc = 0 p = 0 else: valid = False valid_losses = None valid_acces = None # Initialization for monitor if monitor: m = Monitor(monitor_acc=True) train_losses = [] train_acces = [] stop = False iterations = epoch * train_batch_num for iter in range(iterations): i = int(iter / train_batch_num) # current epoch j = iter % train_batch_num # batch index # Train on a batch train_loss = train_fn(j) train_losses.append(train_loss) self.logger.info('TRAINING - Epoch({0:4d} / {1:4d}), train loss: {2}'.format(i + 1, epoch, train_loss)) # Validating if valid and iter % valid_freq == 0: valid_loss = np.mean([valid_fn(k) for k in range(valid_batch_num)]) valid_losses.append(valid_loss) train_acc = train_acc_fn() train_acces.append(train_acc) valid_acc = valid_acc_fn() valid_acces.append(valid_acc) self.logger.info('VALIDATING - Iteration ({0}), valid loss: {1}'.format(iter, valid_loss)) self.logger.info('VALIDATING - Iteration ({0}), train acc: {1}'.format(iter, train_acc)) self.logger.info('VALIDATING - Iteration ({0}), valid acc: {1}'.format(iter, valid_acc)) # Be patient if get lower validation losss if valid_acc > best_valid_acc: best_valid_acc = valid_acc p = 0 else: p += 1 if p >= patience: stop = True if monitor: m.update(train_losses, valid_losses, valid_freq, train_acces, valid_acces) if stop: break if monitor: m.save()
def train(self, train_x, mask_train_x, train_y, mask_train_y, epoch=10, batch_size=128, validation_data=None, valid_freq=100, patience=10, monitor=False, epoch_end_callback=None): # Define the symbolic train model batch_index = T.iscalar('batch_index') x = T.imatrix('x') y = T.imatrix('y') m_x = T.imatrix('m_x') m_y = T.imatrix('m_y') loss = self.loss(x, m_x, y, m_y) updates = self.optimizer.get_updates(loss, self.params) sample_num = train_x.get_value(borrow=True).shape[0] train_batch_num = int(np.ceil(sample_num / batch_size)) train_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: train_x[batch_index * batch_size: (batch_index + 1) * batch_size], y: train_y[batch_index * batch_size: (batch_index + 1) * batch_size], m_x: mask_train_x[:, batch_index * batch_size: (batch_index + 1) * batch_size], m_y: mask_train_y[:, batch_index * batch_size: (batch_index + 1) * batch_size] } ) # Initialization for validation if validation_data is not None: valid = True (valid_x, mask_valid_x, valid_y, mask_valid_y) = validation_data valid_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: valid_x[batch_index * batch_size: (batch_index + 1) * batch_size], y: valid_y[batch_index * batch_size: (batch_index + 1) * batch_size], m_x: mask_valid_x[:, batch_index * batch_size: (batch_index + 1) * batch_size], m_y: mask_valid_y[:, batch_index * batch_size: (batch_index + 1) * batch_size] } ) valid_losses = [] valid_batch_num = int(np.ceil(valid_x.get_value(borrow=True).shape[0] / batch_size)) best_valid_loss = np.inf p = 0 else: valid = False valid_losses = None # Initialization for monitor if monitor: m = Monitor(monitor_acc=False) train_losses = [] stop = False iterations = epoch * train_batch_num for iter in range(iterations): i = int(iter / train_batch_num) # current epoch j = iter % train_batch_num # batch index # Train on a batch train_loss = train_fn(j) train_losses.append(train_loss) self.logger.info('TRAINING - Epoch({0:4d} / {1:4d}), train loss: {2}'.format(i + 1, epoch, train_loss)) if valid and iter % valid_freq == 0: valid_loss = np.mean([valid_fn(k) for k in range(valid_batch_num)]) valid_losses.append(valid_loss) self.logger.info('VALIDATING - Iteration ({0}), valid loss: {1}'.format(iter, valid_loss)) # Be patient if get lower validation losss if valid_loss < best_valid_loss: best_valid_loss = valid_loss p = 0 else: p += 1 if p >= patience: stop = True if monitor: m.update(train_losses, valid_losses, valid_freq) if stop: break if iter % train_batch_num == 0 and epoch_end_callback and callable(epoch_end_callback): epoch_end_callback() if monitor: m.save() return np.asarray(train_losses).reshape(epoch, train_batch_num)
def run_simultrans(model, options_file=None, config=None, policy=None, id=None, remote=False): WORK = config['workspace'] # check hidden folders paths = [ '.policy', '.pretrained', '.log', '.config', '.images', '.translate' ] for p in paths: p = WORK + p if not os.path.exists(p): os.mkdir(p) if id is not None: fcon = WORK + '.config/{}.conf'.format(id) if os.path.exists(fcon): print 'load config files' policy, config = pkl.load(open(fcon, 'r')) # ============================================================================== # # load model model_options # ============================================================================== # _model = model.split('/')[-1] if options_file is not None: with open(options_file, 'rb') as f: options = pkl.load(f) else: with open('%s.pkl' % model, 'rb') as f: options = pkl.load(f) print 'merge configuration into options' for w in config: if (w in options) and (config[w] is not None): options[w] = config[w] print 'load options...' for w, p in sorted(options.items(), key=lambda x: x[0]): print '{}: {}'.format(w, p) # load detail settings from option file: dictionary, dictionary_target = options['dictionaries'] def _iter(fname): with open(fname, 'r') as f: for line in f: words = line.strip().split() x = map(lambda w: word_dict[w] if w in word_dict else 1, words) x = map(lambda ii: ii if ii < options['n_words'] else 1, x) x += [0] yield x def _check_length(fname): f = open(fname, 'r') count = 0 for _ in f: count += 1 f.close() return count # load source dictionary and invert with open(dictionary, 'rb') as f: word_dict = pkl.load(f) word_idict = dict() for kk, vv in word_dict.iteritems(): word_idict[vv] = kk word_idict[0] = '<eos>' word_idict[1] = 'UNK' # load target dictionary and invert with open(dictionary_target, 'rb') as f: word_dict_trg = pkl.load(f) word_idict_trg = dict() for kk, vv in word_dict_trg.iteritems(): word_idict_trg[vv] = kk word_idict_trg[0] = '<eos>' word_idict_trg[1] = 'UNK' ## use additional input for the policy network options['pre'] = config['pre'] # ================================================================================= # # Build a Simultaneous Translator # ================================================================================= # # allocate model parameters params = init_params(options) params = load_params(model, params) tparams = init_tparams(params) # print 'build the model for computing cost (full source sentence).' trng, use_noise, \ _x, _x_mask, _y, _y_mask, \ opt_ret, \ cost, f_cost = build_model(tparams, options) print 'done' # functions for sampler f_sim_ctx, f_sim_init, f_sim_next = build_simultaneous_sampler( tparams, options, trng) # function for finetune if config['finetune'] != 'nope': f_fine_init, f_fine_cost, f_fine_update = build_fine( tparams, options, fullmodel=True if config['finetune'] == 'full' else False) def _translate(src, trg, train=False, samples=config['sample'], greedy=False): ret = simultaneous_decoding( f_sim_ctx, f_sim_init, f_sim_next, f_cost, _policy, src, trg, word_idict_trg, step=config['step'], peek=config['peek'], sidx=config['s0'], n_samples=samples, reward_config={ 'target': config['target'], 'gamma': config['gamma'], 'Rtype': config['Rtype'], 'maxsrc': config['maxsrc'], 'greedy': greedy, 'upper': config['upper'] }, train=train, use_forget=config['forget'], use_newinput=config['pre'], use_coverage=config['coverage'], on_groundtruth=0 if config['finetune'] == 'nope' else 10) return ret # check the ID: policy['base'] = _model _policy = Policy(trng, options, policy, config, n_in=options['readout_dim'] + 1 if config['coverage'] else options['readout_dim'], n_out=3 if config['forget'] else 2, recurrent=policy['recurrent'], id=id) # make the dataset ready for training & validation # train_ = options['datasets'][0] # train_num = _check_length trainIter = TextIterator(options['datasets'][0], options['datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=config['batchsize'], maxlen=options['maxlen']) train_num = trainIter.num validIter = TextIterator(options['valid_datasets'][0], options['valid_datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=64, cache=10, maxlen=1000000) valid_num = validIter.num valid_ = options['valid_datasets'][0] valid_num = _check_length(valid_) print 'training set {} lines / validation set {} lines'.format( train_num, valid_num) print 'use the reward function {}'.format(chr(config['Rtype'] + 65)) # ================================================================================= # # Main Loop: Run # ================================================================================= # print 'Start Simultaneous Translator...' probar = Progbar(train_num / config['batchsize'], with_history=False) monitor = None if remote: monitor = Monitor(root='http://localhost:9000') # freqs save_freq = 200 sample_freq = 10 valid_freq = 200 valid_size = 200 display_freq = 50 finetune_freq = 5 history, last_it = _policy.load() action_space = ['W', 'C', 'F'] Log_avg = {} time0 = timer() pipe = PIPE(['x', 'x_mask', 'y', 'y_mask', 'c_mask']) for it, (srcs, trgs) in enumerate(trainIter): # only one sentence each iteration if it < last_it: # go over the scanned lines. continue # for validation # doing the whole validation!! reference = [] system = [] reference2 = [] system2 = [] if it % valid_freq == 0: print 'start validation' collections = [[], [], [], [], []] probar_v = Progbar(valid_num / 64 + 1) for ij, (srcs, trgs) in enumerate(validIter): # new_srcs, new_trgs = [], [] # for src, trg in zip(srcs, trgs): # if len(src) < config['s0']: # continue # ignore when the source sentence is less than sidx. we don't use the policy\ # else: # new_srcs += [src] # new_trgs += [trg] # if len(new_srcs) == 0: # continue # srcs, trgs = new_srcs, new_trgs statistics = _translate(srcs, trgs, train=False, samples=1, greedy=True) quality, delay, reward = zip(*statistics['track']) reference += statistics['Ref'] system += statistics['Sys'] # print ' '.join(reference[-1][0]) # print ' '.join(system[-1]) # compute the average consective waiting length def _consective(action): waits = [] temp = 0 for a in action: if a == 0: temp += 1 elif temp > 0: waits += [temp] temp = 0 if temp > 0: waits += [temp] mean = numpy.mean(waits) gec = numpy.max( waits) # numpy.prod(waits) ** (1./len(waits)) return mean, gec def _max_length(action): _cur = 0 _end = 0 _max = 0 for it, a in enumerate(action): if a == 0: _cur += 1 elif a == 2: _end += 1 temp = _cur - _end if temp > _max: _max = temp return _max maxlen = [ _max_length(action) for action in statistics['action'] ] means, gecs = zip(*(_consective(action) for action in statistics['action'])) collections[0] += quality collections[1] += delay collections[2] += means collections[3] += gecs collections[4] += maxlen values = [('quality', numpy.mean(quality)), ('delay', numpy.mean(delay)), ('wait_mean', numpy.mean(means)), ('wait_max', numpy.mean(gecs)), ('max_len', numpy.mean(maxlen))] probar_v.update(ij + 1, values=values) validIter.reset() valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx = [ numpy.mean(a) for a in collections ] print 'Iter = {}: AVG BLEU = {}, DELAY = {}, WAIT(MEAN) = {}, WAIT(MAX) = {}, MaxLen={}'.format( it, valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx) print 'Compute the Corpus BLEU={} (greedy)'.format( corpus_bleu(reference, system)) with open(WORK + '.translate/test.txt', 'w') as fout: for sys in system: fout.write('{}\n'.format(' '.join(sys))) with open(WORK + '.translate/ref.txt', 'w') as fout: for ref in reference: fout.write('{}\n'.format(' '.join(ref[0]))) if config['upper']: print 'done' import sys sys.exit(-1) # training set sentence tuning new_srcs, new_trgs = [], [] for src, trg in zip(srcs, trgs): if len(src) <= config['s0']: continue # ignore when the source sentence is less than sidx. we don't use the policy\ else: new_srcs += [src] new_trgs += [trg] if len(new_srcs) == 0: continue srcs, trgs = new_srcs, new_trgs try: statistics, info, pipe_t = _translate(srcs, trgs, train=True) except Exception: print 'translate a empty sentence. bug.' continue # samples, scores, actions, rewards, info, pipe_t = _translate(srcs, trgs, train=True) # print pipe_t if config['finetune'] != 'nope': for idx, act in enumerate(pipe_t['action']): _start = 0 _end = 0 _mask = [0 for _ in srcs[0]] _cmask = [] pipe.messages['x'] += srcs pipe.messages['y'] += [pipe_t['sample'][idx]] for a in act: # print _start, _end if a == 0: _mask[_start] = 1 _start += 1 elif a == 2: _mask[_end] = 0 _end += 1 else: _cmask.append(_mask) # print numpy.asarray(_cmask).shape pipe.messages['c_mask'].append(_cmask) if it % finetune_freq == (finetune_freq - 1): num = len(pipe.messages['x']) max_x = max([len(v) for v in pipe.messages['x']]) max_y = max([len(v) for v in pipe.messages['y']]) xx, xx_mask = _padding(pipe.messages['x'], shape=(max_x, num), return_mask=True, dtype='int64') yy, yy_mask = _padding(pipe.messages['y'], shape=(max_y, num), return_mask=True, dtype='int64') cc_mask = _padding(pipe.messages['c_mask'], shape=(max_y, num, max_x)).transpose([0, 2, 1]) # fine-tune the EncDec of translation if config['finetune'] == 'full': cost = f_fine_cost(xx, xx_mask, yy, yy_mask, cc_mask) elif config['finetune'] == 'decoder': cost = f_fine_cost(xx, xx_mask, yy, yy_mask, cc_mask) else: raise NotImplementedError print '\nIter={} || cost = {}'.format(it, cost[0]) f_fine_update(0.00001) pipe.reset() if it % sample_freq == 0: print '\nModel:{} has been trained for {} hours'.format( _policy.id, (timer() - time0) / 3600.) print 'source: ', _bpe2words(_seqs2words([srcs[0]], word_idict))[0] print 'target: ', _bpe2words(_seqs2words([trgs[0]], word_idict_trg))[0] # obtain the translation results samples = _bpe2words( _seqs2words(statistics['sample'], word_idict_trg)) # obtain the delay (normalized) # delays = _action2delay(srcs[0], statistics['action']) c = 0 for j in xrange(len(samples)): if statistics['secs'][j][0] == 0: if c < 5: c += 1 print '---ID: {}'.format(_policy.id) print 'sample: ', samples[j] # print 'action: ', ','.join( # ['{}({})'.format(action_space[t], f) # for t, f in # zip(statistics['action'][j], statistics['forgotten'][j])]) print 'action: ', ','.join([ '{}'.format(action_space[t]) for t in statistics['action'][j] ]) print 'quality:', statistics['track'][j][0] print 'delay:', statistics['track'][j][1] # print 'score:', statistics['score'][j] break values = [(w, info[w]) for w in info] probar.update(it + 1, values=values) # NaN detector for w in info: if numpy.isnan(info[w]) or numpy.isinf(info[w]): raise RuntimeError, 'NaN/INF is detected!! {} : ID={}'.format( w, id) # remote display if remote: logs = { 'R': info['R'], 'Q': info['Q'], 'D': info['D'], 'P': float(info['P']) } # print logs for w in logs: Log_avg[w] = Log_avg.get(w, 0) + logs[w] if it % display_freq == (display_freq - 1): for w in Log_avg: Log_avg[w] /= display_freq monitor.display(it + 1, Log_avg) Log_avg = dict() # save the history & model history += [info] if it % save_freq == 0: _policy.save(history, it)
def run_simultrans(model, options_file=None, config=None, id=None, remote=False): WORK = config['workspace'] # check hidden folders paths = [ '.policy', '.pretrained', '.log', '.config', '.images', '.translate' ] for p in paths: p = WORK + p if not os.path.exists(p): os.mkdir(p) if id is not None: fcon = WORK + '.config/{}.conf'.format(id) if os.path.exists(fcon): print 'load config files' policy, config = pkl.load(open(fcon, 'r')) # ============================================================================== # # load model model_options # ============================================================================== # _model = model.split('/')[-1] if options_file is not None: with open(options_file, 'rb') as f: options = pkl.load(f) else: with open('%s.pkl' % model, 'rb') as f: options = pkl.load(f) print 'merge configuration into options' for w in config: # if (w in options) and (config[w] is not None): options[w] = config[w] print 'load options...' for w, p in sorted(options.items(), key=lambda x: x[0]): print '{}: {}'.format(w, p) # load detail settings from option file: dictionary, dictionary_target = options['dictionaries'] # load source dictionary and invert with open(dictionary, 'rb') as f: word_dict = pkl.load(f) word_idict = dict() for kk, vv in word_dict.iteritems(): word_idict[vv] = kk word_idict[0] = '<eos>' word_idict[1] = 'UNK' # load target dictionary and invert with open(dictionary_target, 'rb') as f: word_dict_trg = pkl.load(f) word_idict_trg = dict() for kk, vv in word_dict_trg.iteritems(): word_idict_trg[vv] = kk word_idict_trg[0] = '<eos>' word_idict_trg[1] = 'UNK' options['pre'] = config['pre'] # ========================================================================= # # Build a Simultaneous Translator # ========================================================================= # # allocate model parameters params = init_params(options) params = load_params(model, params) tparams = init_tparams(params) # print 'build the model for computing cost (full source sentence).' trng, use_noise, \ _x, _x_mask, _y, _y_mask, \ opt_ret, \ cost, f_cost = build_model(tparams, options) print 'done' # functions for sampler f_sim_ctx, f_sim_init, f_sim_next = build_simultaneous_sampler( tparams, options, trng) # function for finetune the underlying model if options['finetune']: ff_init, ff_cost, ff_update = build_simultaneous_model(tparams, options, rl=True) funcs = [ f_sim_ctx, f_sim_init, f_sim_next, f_cost, ff_init, ff_cost, ff_update ] else: funcs = [f_sim_ctx, f_sim_init, f_sim_next, f_cost] # build a res-predictor if options['predict']: params_act = get_actor('gru')[0](options, prefix='pdt', nin=options['dim']) pass # check the ID: options['base'] = _model agent = Policy(trng, options, n_in=options['readout_dim'] + 1 if options['coverage'] else options['readout_dim'], n_out=3 if config['forget'] else 2, recurrent=options['recurrent'], id=id) # make the dataset ready for training & validation trainIter = TextIterator(options['datasets'][0], options['datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=config['batchsize'], maxlen=options['maxlen']) train_num = trainIter.num validIter = TextIterator(options['valid_datasets'][0], options['valid_datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=20, cache=10, maxlen=1000000) valid_num = validIter.num print 'training set {} lines / validation set {} lines'.format( train_num, valid_num) print 'use the reward function {}'.format(chr(config['Rtype'] + 65)) # ========================================================================== # # Main Loop: Run # ========================================================================== # print 'Start Simultaneous Translator...' monitor = None if remote: monitor = Monitor(root='http://localhost:9000') # freqs save_freq = 200 sample_freq = 10 valid_freq = 200 valid_size = 200 display_freq = 50 finetune_freq = 5 history, last_it = agent.load() action_space = ['W', 'C', 'F'] Log_avg = {} time0 = timer() pipe = OrderedDict() for key in ['x', 'x_mask', 'y', 'y_mask', 'c_mask']: pipe[key] = [] def _translate(src, trg, samples=None, train=False, greedy=False, show=False, full=False): time0 = time.time() if full: options1 = copy.copy(options) options1['upper'] = True else: options1 = options ret = simultaneous_decoding(funcs, agent, options1, src, trg, word_idict_trg, samples, greedy, train) if show: info = ret[1] values = [(w, float(info[w])) for w in info if w != 'advantages'] print ' , '.join(['{}={:.3f}'.format(k, f) for k, f in values]), print '...{}s'.format(time.time() - time0) return ret for it, (srcs, trgs) in enumerate(trainIter): # only one sentence each iteration if it < last_it: # go over the scanned lines. continue # for validation # doing the whole validation!! reference = [] system = [] if it % valid_freq == (valid_freq - 1): print 'start validation' collections = [[], [], [], [], []] probar_v = Progbar(valid_num / 20 + 1) for ij, (srcs, trgs) in enumerate(validIter): statistics = _translate(srcs, trgs, samples=1, train=False, greedy=True) quality, delay, reward = zip(*statistics['track']) reference += statistics['Ref'] system += statistics['Sys'] # compute the average consective waiting length def _consective(action): waits = [] temp = 0 for a in action: if a == 0: temp += 1 elif temp > 0: waits += [temp] temp = 0 if temp > 0: waits += [temp] mean = numpy.mean(waits) gec = numpy.max( waits) # numpy.prod(waits) ** (1./len(waits)) return mean, gec def _max_length(action): _cur = 0 _end = 0 _max = 0 for it, a in enumerate(action): if a == 0: _cur += 1 elif a == 2: _end += 1 temp = _cur - _end if temp > _max: _max = temp return _max maxlen = [ _max_length(action) for action in statistics['action'] ] means, gecs = zip(*(_consective(action) for action in statistics['action'])) collections[0] += quality collections[1] += delay collections[2] += means collections[3] += gecs collections[4] += maxlen values = [('quality', numpy.mean(quality)), ('delay', numpy.mean(delay)), ('wait_mean', numpy.mean(means)), ('wait_max', numpy.mean(gecs)), ('max_len', numpy.mean(maxlen))] probar_v.update(ij + 1, values=values) validIter.reset() valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx = [ numpy.mean(a) for a in collections ] print 'Iter = {}: AVG BLEU = {}, DELAY = {}, WAIT(MEAN) = {}, WAIT(MAX) = {}, MaxLen={}'.format( it, valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx) print 'Compute the Corpus BLEU={} (greedy)'.format( corpus_bleu(reference, system)) with open(WORK + '.translate/test.txt', 'w') as fout: for sys in system: fout.write('{}\n'.format(' '.join(sys))) with open(WORK + '.translate/ref.txt', 'w') as fout: for ref in reference: fout.write('{}\n'.format(' '.join(ref[0]))) history += [collections] print 'done' if options['upper']: print 'done' import sys sys.exit(-1) # training set sentence tuning new_srcs, new_trgs = [], [] for src, trg in zip(srcs, trgs): if len(src) <= options['s0']: continue # ignore when the source sentence is less than sidx. else: new_srcs += [src] new_trgs += [trg] if len(new_srcs) == 0: continue srcs, trgs = new_srcs, new_trgs statistics, info = _translate(srcs, trgs, train=True, show=True) if it % sample_freq == 0: # obtain the translation results samples = _bpe2words( _seqs2words(statistics['sample'], word_idict_trg, statistics['action'], 1)) sources = _bpe2words( _seqs2words(statistics['SWord'], word_idict, statistics['action'], 0)) targets = _bpe2words( _seqs2words(statistics['TWord'], word_idict_trg)) # obtain the delay (normalized) # delays = _action2delay(srcs[0], statistics['action']) c = 0 for j in xrange(len(samples)): if statistics['seq_info'][j][0] == 0: if c < (config['sample'] / 2.): c += 1 continue print '--Iter: {}'.format(it) print 'source: ', sources[j] print 'sample: ', samples[j] print 'target: ', targets[j] print 'quality:', statistics['track'][j][0] print 'delay:', statistics['track'][j][1] print 'reward:', statistics['track'][j][2] break # NaN detector #for w in info: # if numpy.isnan(info[w]) or numpy.isinf(info[w]): # raise RuntimeError, 'NaN/INF is detected!! {} : ID={}'.format(w, id) # remote display if remote: logs = { 'R': info['R'], 'Q': info['Q'], 'D': info['D'], 'P': float(info['P']) } if 'a_cost' in info: logs['A'] = info['a_cost'] print logs for w in logs: Log_avg[w] = Log_avg.get(w, 0) + logs[w] if it % display_freq == (display_freq - 1): for w in Log_avg: Log_avg[w] /= display_freq monitor.display(it + 1, Log_avg) Log_avg = dict() # save the history & model history += [info] if it % save_freq == 0: agent.save(history, it)
def train(self, train_x, train_y, epoch=100, batch_size=128, validation_data=None, valid_freq=100, patience=10, monitor=False): # Define the symbolic train model batch_index = T.iscalar('batch_index') x = T.matrix('x', dtype=train_x.get_value(borrow=True).dtype) y = T.vector('y', dtype=train_y.get_value(borrow=True).dtype) loss = self.loss(x, y) updates = self.optimizer.get_updates(loss, self.params) sample_num = train_x.get_value(borrow=True).shape[0] train_batch_num = int(np.ceil(sample_num / batch_size)) train_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: train_x[batch_index * batch_size:(batch_index + 1) * batch_size], y: train_y[batch_index * batch_size:(batch_index + 1) * batch_size] }) train_acc_fn = theano.function([], self.score(train_x, train_y)) # Initialization for validation if validation_data is not None: valid = True (valid_x, valid_y) = validation_data valid_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: valid_x[batch_index * batch_size:(batch_index + 1) * batch_size], y: valid_y[batch_index * batch_size:(batch_index + 1) * batch_size] }) valid_acc_fn = theano.function([], self.score(valid_x, valid_y)) valid_batch_num = int( np.ceil(valid_x.get_value(borrow=True).shape[0] / batch_size)) valid_losses = [] valid_acces = [] best_valid_acc = 0 p = 0 else: valid = False valid_losses = None valid_acces = None # Initialization for monitor if monitor: m = Monitor(monitor_acc=True) train_losses = [] train_acces = [] stop = False iterations = epoch * train_batch_num for iter in range(iterations): i = int(iter / train_batch_num) # current epoch j = iter % train_batch_num # batch index # Train on a batch train_loss = train_fn(j) train_losses.append(train_loss) self.logger.info( 'TRAINING - Epoch({0:4d} / {1:4d}), train loss: {2}'.format( i + 1, epoch, train_loss)) # Validating if valid and iter % valid_freq == 0: valid_loss = np.mean( [valid_fn(k) for k in range(valid_batch_num)]) valid_losses.append(valid_loss) train_acc = train_acc_fn() train_acces.append(train_acc) valid_acc = valid_acc_fn() valid_acces.append(valid_acc) self.logger.info( 'VALIDATING - Iteration ({0}), valid loss: {1}'.format( iter, valid_loss)) self.logger.info( 'VALIDATING - Iteration ({0}), train acc: {1}'.format( iter, train_acc)) self.logger.info( 'VALIDATING - Iteration ({0}), valid acc: {1}'.format( iter, valid_acc)) # Be patient if get lower validation losss if valid_acc > best_valid_acc: best_valid_acc = valid_acc p = 0 else: p += 1 if p >= patience: stop = True if monitor: m.update(train_losses, valid_losses, valid_freq, train_acces, valid_acces) if stop: break if monitor: m.save()
def train(self, train_x, mask_train_x, train_y, mask_train_y, epoch=10, batch_size=128, validation_data=None, valid_freq=100, patience=10, monitor=False, epoch_end_callback=None): # Define the symbolic train model batch_index = T.iscalar('batch_index') x = T.imatrix('x') y = T.imatrix('y') m_x = T.imatrix('m_x') m_y = T.imatrix('m_y') loss = self.loss(x, m_x, y, m_y) updates = self.optimizer.get_updates(loss, self.params) sample_num = train_x.get_value(borrow=True).shape[0] train_batch_num = int(np.ceil(sample_num / batch_size)) train_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: train_x[batch_index * batch_size:(batch_index + 1) * batch_size], y: train_y[batch_index * batch_size:(batch_index + 1) * batch_size], m_x: mask_train_x[:, batch_index * batch_size:(batch_index + 1) * batch_size], m_y: mask_train_y[:, batch_index * batch_size:(batch_index + 1) * batch_size] }) # Initialization for validation if validation_data is not None: valid = True (valid_x, mask_valid_x, valid_y, mask_valid_y) = validation_data valid_fn = theano.function( inputs=[batch_index], outputs=loss, updates=updates, givens={ x: valid_x[batch_index * batch_size:(batch_index + 1) * batch_size], y: valid_y[batch_index * batch_size:(batch_index + 1) * batch_size], m_x: mask_valid_x[:, batch_index * batch_size:(batch_index + 1) * batch_size], m_y: mask_valid_y[:, batch_index * batch_size:(batch_index + 1) * batch_size] }) valid_losses = [] valid_batch_num = int( np.ceil(valid_x.get_value(borrow=True).shape[0] / batch_size)) best_valid_loss = np.inf p = 0 else: valid = False valid_losses = None # Initialization for monitor if monitor: m = Monitor(monitor_acc=False) train_losses = [] stop = False iterations = epoch * train_batch_num for iter in range(iterations): i = int(iter / train_batch_num) # current epoch j = iter % train_batch_num # batch index # Train on a batch train_loss = train_fn(j) train_losses.append(train_loss) self.logger.info( 'TRAINING - Epoch({0:4d} / {1:4d}), train loss: {2}'.format( i + 1, epoch, train_loss)) if valid and iter % valid_freq == 0: valid_loss = np.mean( [valid_fn(k) for k in range(valid_batch_num)]) valid_losses.append(valid_loss) self.logger.info( 'VALIDATING - Iteration ({0}), valid loss: {1}'.format( iter, valid_loss)) # Be patient if get lower validation losss if valid_loss < best_valid_loss: best_valid_loss = valid_loss p = 0 else: p += 1 if p >= patience: stop = True if monitor: m.update(train_losses, valid_losses, valid_freq) if stop: break if iter % train_batch_num == 0 and epoch_end_callback and callable( epoch_end_callback): epoch_end_callback() if monitor: m.save() return np.asarray(train_losses).reshape(epoch, train_batch_num)
print( "\n[info] Creating generator and discriminator architectures for GAN...\n" ) gen = Generator(num_classes, image_size, bn=True) disc = Discriminator(num_classes, image_size, min_neurons, bn_epsilon=1e-5) print("\n[info] Pre-training or loading pre-trained discriminator...\n") disc.pretrain(train_gen, valid_gen, pretrain_iterations, pretrain_learning_rate, retrain=False) ''' start training GAN for real ''' print("\n[info] Start training GAN...\n") trial_name = 'GANGoghBN' monitor = Monitor(trial_name) checkpoint = 0 if checkpoint > 0: success = gen.load('{}{}'.format(trial_name, checkpoint)) success &= disc.load('{}{}'.format(trial_name, checkpoint)) if not success: checkpoint = 0 gen_opt = tf.keras.optimizers.Adam(learning_rate=gen_learning_rate, beta_1=.5, beta_2=.9) disc_opt = tf.keras.optimizers.Adam(learning_rate=disc_learning_rate, beta_1=.5, beta_2=.9) for i in range(checkpoint, train_iterations):