def compute_perplexity(model, sess, name): """Compute perplexity of the output of the model. Args: model: model for compute perplexity. sess: tensorflow session to use. name: name of the batch. Returns: The perplexity of the eval outputs. """ total_loss = 0 total_predict_count = 0 start_time = time.time() step = 0 while True: try: loss, predict_count, batch_size = model.eval(sess) total_loss += loss * batch_size total_predict_count += predict_count step += 1 if step % 500 == 0: ls = total_loss / total_predict_count ppl = misc.safe_exp(ls) print_out(" ## After %d steps, loss %.2f - ppl %.3f" % (step, ls, ppl)) except tf.errors.OutOfRangeError: break perplexity = safe_exp(total_loss / total_predict_count) print_time(" eval %s: perplexity %.2f" % (name, perplexity), start_time) return perplexity
def load_embedding_from_path(args): """Load a TokenEmbedding.""" if 'bin' in args.embedding_path: with utils.print_time('load fastText model.'): model = \ nlp.model.train.FasttextEmbeddingModel.load_fasttext_format( args.embedding_path) embedding = nlp.embedding.TokenEmbedding( unknown_token=None, unknown_lookup=model, allow_extend=True, unknown_autoextend=True) idx_to_token = sorted(model._token_to_idx, key=model._token_to_idx.get) if not args.analogy_datasets: # Prune tokens not used in evaluation datasets eval_tokens_ = set( evaluation.get_tokens_in_evaluation_datasets(args)) idx_to_token = [t for t in idx_to_token if t in eval_tokens_] if args.max_vocab_size: idx_to_token = idx_to_token[:args.max_vocab_size] with utils.print_time('compute vectors from subwords ' 'for {} words.'.format(len(idx_to_token))): embedding[idx_to_token] = model[idx_to_token] else: embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path) return embedding
def compute_perplexity(model, sess, name): """Compute perplexity of the output of the model. Args: model: model for compute perplexity. sess: tensorflow session to use. name: name of the batch. Returns: The perplexity of the eval outputs. """ total_loss = 0 total_predict_count = 0 start_time = time.time() while True: try: output_tuple = model.eval(sess) total_loss += output_tuple.eval_loss * output_tuple.batch_size total_predict_count += output_tuple.predict_count except tf.errors.OutOfRangeError: break perplexity = utils.safe_exp(total_loss / total_predict_count) utils.print_time(" eval %s: perplexity %.2f" % (name, perplexity), start_time) return perplexity
def analyze(self): trf = 'transforms.trf' cfg = self.cfg input_video = cfg.convey.curr_input_video vidstab_dir = cfg.convey.curr_vidstab_dir step = 'stepsize='+str(cfg.args.vs_stepsize) mincontrast = float(cfg.args.vs_mincontrast) detect = 'vidstabdetect=shakiness=10:accuracy=15:{0}:mincontrast={1}:result={2}:show=1' filts = detect.format(step, mincontrast, trf) show_detect = path.join(vidstab_dir, 'show.mkv') #skip_first = "select='gte(n,0)'," cmd = ['ffmpeg', '-i', input_video, '-c:v', 'libx264', '-crf', '18', '-vf', filts, '-an', '-y', '-loglevel', 'error', '-stats', show_detect] print('Analyze cam motions in video (libvidstab)') if cfg.args.verbose: print(' '.join(cmd)) s = dt.datetime.now() run(cmd, cwd=vidstab_dir) e = dt.datetime.now() - s utils.print_time(e.total_seconds()) ## saves global_motions.trf ## was needed before gradient descent impl in py from C self.save_global_motions_trf_file(input_video, vidstab_dir)
def analyze2(self): trf = 'transforms.trf' cfg = self.cfg if utils.args_rolling_shutter(): self.create_processed_vidstab_input(cfg.input_processed_video_path) step = 'stepsize='+str(cfg.args.vs_stepsize) mincontrast = float(cfg.args.vs_mincontrast) detect = 'vidstabdetect=shakiness=10:accuracy=15:{0}:mincontrast={1}:result={2}:show=1' filts = detect.format(step, mincontrast, trf) input_video = cfg.convey.curr_input_video vidstab_dir = cfg.convey.curr_vidstab_dir show_detect = path.join(vidstab_dir, 'show.mkv') cmd = ['ffmpeg', '-i', input_video, '-c:v', 'libx264', '-crf', '18', '-vf', filts, '-an', '-y', '-loglevel', 'error', '-stats', show_detect] s = dt.datetime.now() print('Vidstab analyze video after rolling shutter correction') if cfg.args.verbose: print(' '.join(cmd)) print('FFMPEG output:') run(cmd, cwd=vidstab_dir) else: run(cmd, cwd=vidstab_dir, stdout=DEVNULL) e = dt.datetime.now() - s utils.print_time(e.total_seconds())
def train(self, epochs_to_train=5): meta = self.get_meta() ut.print_time('train started: \n%s' % ut.to_file_name(meta)) ut.configure_folders(FLAGS, meta) self.fetch_datasets(self._activation) self.build_model() self._register_training_start() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) self._saver = tf.train.Saver() if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()): self._saver.restore(sess, self.get_checkpoint_path()) ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31) # MAIN LOOP for current_epoch in xrange(epochs_to_train): start = time.time() feed = self._get_epoch_dataset() for _, batch in enumerate(feed): encoding, reconstruction, loss, _, _ = sess.run( [self._encode, self._decode, self._reco_loss, self._train, self._step], feed_dict={self._input: batch[0], self._reconstruction: batch[0]}) self._register_batch(loss) self._register_epoch(current_epoch, epochs_to_train, time.time()-start, sess) self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph) meta = self._register_training() return meta, self._stats['epoch_accuracy']
def load_embedding_from_path(args): """Load a TokenEmbedding.""" if 'bin' in args.embedding_path: with utils.print_time('load fastText model.'): model = \ nlp.model.train.FasttextEmbeddingModel.load_fasttext_format( args.embedding_path) # Add OOV words if the token_embedding can impute them token_set = set() token_set.update( filter(lambda x: x in model, evaluation.get_tokens_in_evaluation_datasets(args))) # OOV words will be imputed and added to the # token_embedding.idx_to_token etc. with utils.print_time('compute vectors from subwords ' 'for {} words.'.format(len(token_set))): embedding = nlp.embedding.TokenEmbedding(unknown_token=None, allow_extend=True) idx_to_tokens = list(token_set) embedding[idx_to_tokens] = model[idx_to_tokens] else: embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path) return embedding
def frames(self): cfg = self.cfg pto_files = sorted(os.listdir(cfg.convey.curr_hugin_ptos_dir)) all_out_frames = cfg.convey.all_out_frames tasks = [] for i, pto in enumerate(pto_files): tasks.append( datatypes.hugin_task(str(i + 1).zfill(6) + '.jpg', pto)) self.prjn_frames_total = len(tasks) self.prjn_frames_cnt = 0 s = dt.datetime.now() with Pool(int(cfg.args.num_cpus)) as p: print(f'Create stabilized frames for output video.') results = [ p.apply_async(hugin.frames_output, args=(t, all_out_frames, cfg.frames_stabilized), callback=self.prjn_worker_callback) for t in tasks ] for r in results: r.get() e = dt.datetime.now() - s utils.print_time(e.total_seconds(), fps=True)
def load_embedding_from_path(args): """Load a TokenEmbedding.""" if args.embedding_path.endswith('.bin'): with utils.print_time('load fastText model.'): model = \ nlp.model.train.FasttextEmbeddingModel.load_fasttext_format( args.embedding_path) idx_to_token = sorted(model._token_to_idx, key=model._token_to_idx.get) embedding = nlp.embedding.TokenEmbedding( unknown_token=None, unknown_lookup=model, allow_extend=True) # Analogy task is open-vocabulary, so must keep all known words. # But if not evaluating analogy, no need to precompute now as all # words for closed vocabulary task can be obtained via the unknown # lookup if not args.analogy_datasets: idx_to_token = [] elif args.analogy_datasets and args.analogy_max_vocab_size: idx_to_token = idx_to_token[:args.analogy_max_vocab_size] embedding['<unk>'] = mx.nd.zeros(model.weight.shape[1]) if idx_to_token: with utils.print_time('compute vectors for {} known ' 'words.'.format(len(idx_to_token))): embedding[idx_to_token] = model[idx_to_token] else: embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path) return embedding
def zscore_signal(writer, original, save_params): # Retrieve/generate zscore params if os.path.isfile(save_params): params = np.load(save_params) normalizer = SignalNormalizer("zscore", mu=params["mu"], std=params["std"]) else: mu = np.mean(original) std = np.std(original) normalizer = SignalNormalizer("zscore", save_params=save_params, mu=mu, std=std) print_time( "mu={:.4f}, std={:.4f} obtained from {}".format( normalizer.mu, normalizer.std, save_params), start_time, ) zscore = normalizer.normalize(original) # Save zscored signal to disk. for path in writer.paths: path = path.replace(".npz", ".threshold{:.0e}.npz".format(args.threshold)) batch = dict(np.load(path)) batch["zscore"] = zscore[batch["start"]:batch["stop"]] np.savez_compressed(path, **batch)
def main(args): utils.print_time(start=True) args.path.checkpoint, args.wandb.notes = utils.make_directories(args) if args.wandb.log: wandb.init(project=args.wandb.project, name=args.wandb.notes, config=args.config, notes=args.wandb.notes, allow_val_change=True) utils.save_code(args) print('=' * 100) print('Arguments =') for arg in vars(args): print('\t' + arg + ':', getattr(args, arg)) print('=' * 100) for n in range(args.train.num_runs): args.seed = n + 1 args.experiment.memory_budget = int(args.experiment.memory_budget) args.path.output = 'Run_{}_{}.txt'.format(n + 1, args.wandb.notes) if args.wandb.log: wandb.config.update(args, allow_val_change=True) print(">" * 30, "Run #", n + 1) run(args, n) print("All Done! ") print('[Elapsed time = {:.1f} min - {:0.1f} hours]'.format( (time.time() - tstart) / (60), (time.time() - tstart) / (3600))) utils.print_time(start=False)
def _register_training(self): best_acc = np.min(self._stats['epoch_accuracy']) meta = self.get_meta() meta['acu'] = int(best_acc) meta['e'] = self.get_past_epochs() ut.print_time('Best Quality: %f for %s' % (best_acc, ut.to_file_name(meta))) return meta
def main(args): utils.make_directories(args) utils.some_sanity_checks(args) utils.save_code(args) print('=' * 100) print('Arguments =') for arg in vars(args): print('\t' + arg + ':', getattr(args, arg)) print('=' * 100) accuracies, forgetting = [], [] for n in range(args.num_runs): args.seed = n args.output = '{}_{}_tasks_seed_{}.txt'.format(args.experiment, args.ntasks, args.seed) print ("args.output: ", args.output) print (" >>>> Run #", n) acc, bwt = run(args, n) accuracies.append(acc) forgetting.append(bwt) print('*' * 100) print ("Average over {} runs: ".format(args.num_runs)) print ('AVG ACC: {:5.4f}% \pm {:5.4f}'.format(np.array(accuracies).mean(), np.array(accuracies).std())) print ('AVG BWT: {:5.2f}% \pm {:5.4f}'.format(np.array(forgetting).mean(), np.array(forgetting).std())) print ("All Done! ") print('[Elapsed time = {:.1f} min]'.format((time.time()-tstart)/(60))) utils.print_time()
def _decode_inference_indices(model, sess, output_infer, output_infer_summary_prefix, inference_indices, tgt_eos, subword_option): """Decoding only a specific set of sentences.""" utils.print_out(" decoding to output %s , num sents %d." % (output_infer, len(inference_indices))) start_time = time.time() with codecs.getwriter("utf-8")(tf.gfile.GFile(output_infer, mode="wb")) as trans_f: trans_f.write("") # Write empty string to ensure file is created. for decode_id in inference_indices: nmt_outputs, infer_summary = model.decode(sess) # get text translation assert nmt_outputs.shape[0] == 1 translation = nmt_utils.get_translation( nmt_outputs, sent_id=0, tgt_eos=tgt_eos, subword_option=subword_option) if infer_summary is not None: # Attention models image_file = output_infer_summary_prefix + str( decode_id) + ".png" utils.print_out(" save attention image to %s*" % image_file) image_summ = tf.Summary() image_summ.ParseFromString(infer_summary) with tf.gfile.GFile(image_file, mode="w") as img_f: img_f.write(image_summ.value[0].image.encoded_image_string) trans_f.write("%s\n" % translation) utils.print_out(translation + b"\n") utils.print_time(" done", start_time)
def get_train_data(args): """Helper function to get training data.""" with print_time('load training dataset'): dataset = nlp.data.Text8(segment='train') with print_time('count tokens'): counter = nlp.data.count_tokens(itertools.chain.from_iterable(dataset)) vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None, bos_token=None, eos_token=None, min_freq=5) idx_to_counts = mx.nd.array([counter[w] for w in vocab.idx_to_token]) negatives_weights = idx_to_counts**0.75 negatives_sampler = nlp.data.UnigramCandidateSampler( weights=negatives_weights) # Skip "unknown" tokens with print_time('code dataset'): coded_dataset = [[ vocab[token] for token in sentence if token in vocab ] for sentence in dataset] with print_time('prune frequent words from sentences'): frequent_tokens_subsampling_constant = 1e-3 f = idx_to_counts / mx.nd.sum(idx_to_counts) idx_to_pdiscard = ( mx.nd.sqrt(frequent_tokens_subsampling_constant / f) + frequent_tokens_subsampling_constant / f).asnumpy() prune_sentences_ = functools.partial(prune_sentences, idx_to_pdiscard=idx_to_pdiscard) coded_dataset = list(map(prune_sentences_, coded_dataset)) with print_time('prepare subwords'): subword_function = nlp.vocab.create_subword_function( 'NGramHashes', ngrams=args.ngrams, num_subwords=args.ngram_buckets) # Precompute a idx to subwordidxs mapping to support fast lookup idx_to_subwordidxs = list(subword_function(vocab.idx_to_token)) max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs) # Padded max_subwordidxs_len + 1 so each row contains at least one -1 # element which can be found by np.argmax below. idx_to_subwordidxs = np.stack( np.pad(b.asnumpy(), (0, max_subwordidxs_len - len(b) + 1), \ constant_values=-1, mode='constant') for b in idx_to_subwordidxs).astype(np.float32) idx_to_subwordidxs = mx.nd.array(idx_to_subwordidxs) logging.info( 'Using %s to obtain subwords. ' 'The word with largest number of subwords ' 'has %s subwords.', subword_function, max_subwordidxs_len) return (coded_dataset, negatives_sampler, vocab, subword_function, idx_to_subwordidxs)
def get_train_data(args): """Helper function to get training data.""" with print_time('load training dataset'): dataset = nlp.data.Text8(segment='train') with print_time('count tokens'): counter = nlp.data.count_tokens(itertools.chain.from_iterable(dataset)) vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None, bos_token=None, eos_token=None, min_freq=5) idx_to_counts = np.array([counter[w] for w in vocab.idx_to_token]) negatives_weights = idx_to_counts**0.75 negatives_sampler = nlp.data.UnigramCandidateSampler( weights=mx.nd.array(negatives_weights)) # Skip "unknown" tokens with print_time('code dataset'): coded_dataset = [[ vocab[token] for token in sentence if token in vocab ] for sentence in dataset] coded_dataset = [ sentence for sentence in coded_dataset if len(sentence) ] with print_time('prune frequent words from sentences'): f = idx_to_counts / np.sum(idx_to_counts) idx_to_pdiscard = 1 - np.sqrt(args.frequent_token_subsampling / f) prune_sentences_ = functools.partial(prune_sentences, idx_to_pdiscard=idx_to_pdiscard) coded_dataset = list(map(prune_sentences_, coded_dataset)) if args.ngram_buckets: # Fasttext model with print_time('prepare subwords'): subword_function = nlp.vocab.create_subword_function( 'NGramHashes', ngrams=args.ngrams, num_subwords=args.ngram_buckets) # Store subword indices for all words in vocabulary idx_to_subwordidxs = list(subword_function(vocab.idx_to_token)) get_subwords_masks = get_subwords_masks_factory(idx_to_subwordidxs) max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs) if max_subwordidxs_len > 500: warnings.warn( 'The word with largest number of subwords ' 'has {} subwords, suggesting there are ' 'some noisy words in your vocabulary. ' 'You should filter out very long words ' 'to avoid memory issues.'.format(max_subwordidxs_len)) return (coded_dataset, negatives_sampler, vocab, subword_function, get_subwords_masks) else: return coded_dataset, negatives_sampler, vocab
def compute_hugin_camera_rotations_processed(self): cfg = self.cfg if cfg.args.vidstab_prjn > -1: vidstab_dir = cfg.prjn_dir2_vidstab_prjn else: vidstab_dir = cfg.prjn_dir2_vidstab_orig if not utils.to_upd_camera_rotations_processed(vidstab_dir): return frames_dir = cfg.frames_input_processed imgs = sorted(os.listdir(frames_dir)) self.canv_half_hfov = math.radians(self.rectilinear_pto.canv_half_hfov) max_horizont_tan = math.tan(self.canv_half_hfov) self.tan_pix = max_horizont_tan / (self.rectilinear_pto.canvas_w / 2) ## # f_motions = open(path.join(vidstab_dir, 'global_motions.trf')) # transforms_rel = utils.get_global_motions(f_motions) s = dt.datetime.now() transforms_rel = utils.parseTransformsTrf( path.join(vidstab_dir, 'transforms.trf')) e = dt.datetime.now() - s utils.print_time(e.total_seconds(), fps=True) ## transforms_abs = utils.convert_relative_transforms_to_absolute( transforms_rel) transforms_abs_filtered = utils.gauss_filter(cfg.fps, transforms_abs, cfg.args.smoothing) self.pto_txt = utils.create_pto_txt_one_image(cfg.pto.filepath) tasks = [] for i, t in enumerate(zip(transforms_abs_filtered, transforms_rel)): path_img = path.join(frames_dir, imgs[i]) tasks.append((t[0], path_img, t[1])) utils.delete_files_in_dir(cfg.hugin_projects_processed) self.prjn_frames_total = len(tasks) self.prjn_frames_cnt = 0 s = dt.datetime.now() with Pool(int(cfg.args.num_cpus)) as p: print('Rolling shutter, create Hugin pto files with camera moves.') results = [ p.apply_async(self.camera_moves_processed_worker, args=(t, ), callback=self.prjn_worker_callback) for t in tasks ] for r in results: r.get() e = dt.datetime.now() - s utils.print_time(e.total_seconds(), fps=True)
def merge_signal(writer, seq_length): original = np.empty((writer.counter, seq_length)) original[:] = np.NaN for path in writer.paths: batch = np.load(path) print_time("[{}:{}]".format(batch["start"], batch["stop"]), start_time) original[batch["start"] : batch["stop"]] = batch["original"] return original
def train(self, epochs_to_train=5): meta = self.get_meta() ut.print_time('train started: \n%s' % ut.to_file_name(meta)) # return meta, np.random.randn(epochs_to_train) ut.configure_folders(FLAGS, meta) self._dataset, self._filters = self.fetch_datasets(self._activation) self.build_model() self._register_training_start() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) self._saver = tf.train.Saver() if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()): self._saver.restore(sess, self.get_checkpoint_path()) ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31) # MAIN LOOP for current_epoch in xrange(epochs_to_train): feed, permutation = self._get_epoch_dataset() for _, batch in enumerate(feed): filter = batch[1][0] assert batch[1][0, 0] == batch[1][-1, 0] encoding, = sess.run([self._encode], feed_dict={self._input: batch[0] }) # 1.1 encode forward clamped_enc, vae_grad = _clamp(encoding, filter) # 1.2 # clamp sess.run(self._assign_clamped, feed_dict={self._clamped: clamped_enc}) reconstruction, loss, clamped_gradient, _ = sess.run( # 2.1 decode forward+backward [ self._decode, self._decoder_loss, self._clamped_grad, self._train_decoder ], feed_dict={ self._clamped: clamped_enc, self._reconstruction: batch[0] }) declamped_grad = _declamp_grad( vae_grad, clamped_gradient, filter) # 2.2 prepare gradient _, step = sess.run( # 3.0 encode backward path [self._train_encoder, self._step], feed_dict={self._input: batch[0], self._encoding: encoding-declamped_grad}) # Profit self._register_batch(batch, encoding, reconstruction, loss) self._register_epoch(current_epoch, epochs_to_train, permutation, sess) self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph) meta = self._register_training() return meta, self._stats['epoch_accuracy']
def _register_training(self): best_acc = np.min(self._stats['epoch_accuracy']) meta = self.get_meta() meta['acu'] = int(best_acc) meta['e'] = self.get_past_epochs() ut.print_time('Best Quality: %f for %s' % (best_acc, ut.to_file_name(meta))) self.summary_writer.close() return meta
def __call__(self): batch = self.data_iter.get_batch() g_st = time.time() self.grad_fn(*batch) g_ed = time.time() if self.state['lr_adapt'] == 1: if self.step > self.state['lr_adapt_start']: self.lr = self.state['lr0'] /\ (1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta']) self.state['lr'] = float(self.lr) e_st = time.time() old_cost = self.compute_old_cost(*batch) new_cost, error = self.compute_new_cost(self.lr, *batch) rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr) if new_cost > old_cost: print ('Error increasing !? ') self.lr = self.lr / 2. while (numpy.isnan(new_cost) or numpy.isinf(new_cost)): raise Exception('Got Inf/NaN !') self.old_cost = new_cost self.update_params(self.lr) e_ed = time.time() msg = ('.. iter %4d cost %.3g (before update %.3g), error %.3g step_size %.3g ' 'rho %.3g ' 'norm grad %.3g ' 'time [grad] %s,' '[updates param] %s,' 'whole time %s' ) print msg % ( self.step, new_cost, old_cost, error, self.lr, rho, norm_grad, print_time(g_ed - g_st), print_time(e_ed - e_st), print_time(time.time() - self.step_timer) ) self.step_timer = time.time() self.step += 1 ret = { 'cost': float(new_cost), 'error': float(error), 'time_grads': float(g_ed - g_st), 'time_eval': float(e_ed - e_st), 'norm_grad':norm_grad, 'lr': self.lr, 'rho': numpy.float32(rho)} return ret
def merge_signal(writer, seq_length): original = np.empty((writer.counter, seq_length, NUM_FEATURES)) original[:] = np.NaN for path in writer.paths: path = path.replace(".npz", ".threshold{:.0e}.npz".format(args.threshold)) batch = np.load(path) print_time("[{}:{}]".format(batch["start"], batch["stop"]), start_time) original[batch["start"]:batch["stop"]] = batch["original"] return original
def main(): cmd_args, unknown_args = parser.parse_known_args() cfg = config.cfg = config.Configuration(cmd_args) inframes = cfg.inframes = InFrames(cfg) vs = cfg.vidstab = Vidstab(cfg) convey = cfg.convey = Conveyor(cfg) out_frms = cfg.out_frms = out_frames.OutFrames(cfg) ## Cached previous command arguments if not path.exists(cfg.cmd_args): cmd_args.force_upd = True open(cfg.cmd_args, 'w').write('\n'.join(sys.argv[1:])) args_list = open(cfg.cmd_args, 'r').read().splitlines() prev_parser = ArgumentParser(description='Previous arguments') args.init_cmd_args(prev_parser) cfg.prev_args, unknown_args_prev = prev_parser.parse_known_args( args=args_list) open(cfg.cmd_args, 'w').write('\n'.join(sys.argv[1:])) # update prev args add_step = convey.add_step step_arg = int(cmd_args.step) print('Videofile: ', cfg.args.videofile) print('Workdir: ', cfg.args.workdir) print('PTO(Hugin): ', cfg.args.pto_name) add_step('STEP_1: input frames and split audio.', convey.to_upd_input_frames, inframes.store_input_frames) add_step('STEP_2: analyze cam motions in input video.', convey.to_upd_analyze, vs.analyze) add_step('STEP_3: camera rotations in Hugin.', convey.to_upd_camera_rotations, out_frms.compute_hugin_camera_rotations) add_step('STEP_4: analyze cam moves with corrected Rolling Shutter.', convey.to_upd_rs_analyze, convey.rs_analyze) add_step('STEP_5: create stabilized frames, Hugin.', convey.to_upd_out_frames, out_frms.frames) add_step('STEP_6: create video from stabilized frames.', convey.to_upd_out_video, out_frms.video) s = dt.datetime.now() convey.execute(step_arg) e = dt.datetime.now() - s print() print() utils.print_time(e.total_seconds(), prefix='Total time')
def apply_gaussian(images, sigma): if sigma == 0: return images # ut.print_time('start') res = images.copy() for i, image in enumerate(res): for channel in range(image.shape[-1]): image[:, :, channel] = filters.gaussian_filter(image[:, :, channel], sigma) ut.print_time('finish s:%.1f' % sigma, images[2,10,10,0], res[2,10,10,0]) return res
async def scf_handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): bridge = Conn("Bridge", reader, writer) uid = await bridge.read(4) uid = uid.decode("ascii") client = uid_socket[uid] bridge.target = client.target bridge_addr, _ = bridge.writer.get_extra_info("peername") print_time(f"Tencent IP:{bridge_addr} <=> {client.target} established") await socks5_forward(client, bridge)
def visualize_encodings(encodings, file_name=None, grid=None, skip_every=999, fast=False, fig=None, interactive=False): encodings = manual_pca(encodings) if encodings.shape[1] <= 3: return print_data_only(encodings, file_name, fig=fig, interactive=interactive) encodings = encodings[0:720] hessian_euc = dist.squareform(dist.pdist(encodings[0:720], 'euclidean')) hessian_cos = dist.squareform(dist.pdist(encodings[0:720], 'cosine')) grid = (3, 4) if grid is None else grid project_ops = [] n = 2 project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa'))) project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified'))) project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) project_ops.append(("TSNE 30/2000 N:%d" % n, TSNE(perplexity=30, n_components=n, init='pca', n_iter=2000))) n = 3 project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa'))) project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified'))) project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) project_ops.append(('MDS cosine N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) plot_places = [] for i in range(12): u, v = int(i / (skip_every - 1)), i % (skip_every - 1) j = v + u * skip_every + 1 plot_places.append(j) fig = get_figure(fig) fig.set_size_inches(fig.get_size_inches()[0] * grid[0] / 1., fig.get_size_inches()[1] * grid[1] / 2.0) for i, (name, manifold) in enumerate(project_ops): is3d = 'N:3' in name try: if is3d: subplot = plt.subplot(grid[0], grid[1], plot_places[i], projection='3d') else: subplot = plt.subplot(grid[0], grid[1], plot_places[i]) data_source = encodings if not _needs_hessian(manifold) else \ (hessian_cos if 'cosine' in name else hessian_euc) projections = manifold.fit_transform(data_source) scatter(subplot, projections, is3d, _build_radial_colors(len(data_source))) subplot.set_title(name) except: print(name, "Unexpected error: ", sys.exc_info()[0], sys.exc_info()[1] if len(sys.exc_info()) > 1 else '') visualize_data_same(encodings, grid=grid, places=plot_places[-4:]) if not interactive: save_fig(file_name, fig) ut.print_time('visualization finished')
def print_epoch_info(self, accuracy, current_epoch, epochs, elapsed): epochs_past = self.get_past_epochs() - current_epoch accuracy_info = '' if accuracy is None else '| accuracy %d' % int( accuracy) epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past - 1) epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs, epoch_past_info) time_info = '%2dms/bt' % (elapsed / FLAGS.epoch_size * 1000) info_string = ' '.join([epoch_count, accuracy_info, time_info]) ut.print_time(info_string, same_line=True)
def print_epoch_info(self, accuracy, current_epoch, epochs, elapsed): epochs_past = self.get_past_epochs() - current_epoch accuracy_info = '' if accuracy is None else '| accuracy %d' % int(accuracy) epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past - 1) epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs, epoch_past_info) time_info = '%2dms/bt' % (elapsed / FLAGS.epoch_size*1000) info_string = ' '.join([ epoch_count, accuracy_info, time_info]) ut.print_time(info_string, same_line=True)
def save_predictions_from_model(dicModel, run_name='all_models', output_folder=PATH_OUTPUT): # create the filename filename = '{}pred_{}_{}_{}.csv'.\ format(output_folder, run_name, int(dicModel['score_val'] * 1000), dicModel['model'].__class__.__name__) #save the predictions pd.DataFrame(dict(preds=dicModel['pred_val'])) \ .to_csv(filename, index=False) print_time('Saved {}'.format(filename))
def __call__(self): batch = self.data_iter.get_batch() g_st = time.time() self.grad_fn(*batch) g_ed = time.time() if self.state['lr_adapt'] == 1: if self.step > self.state['lr_adapt_start']: self.lr = self.state['lr0'] /\ (1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta']) self.state['lr'] = float(self.lr) e_st = time.time() old_cost = self.compute_old_cost(*batch) new_cost, error = self.compute_new_cost(self.lr, *batch) rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr) if new_cost > old_cost: print('Error increasing !? ') self.lr = self.lr / 2. while (numpy.isnan(new_cost) or numpy.isinf(new_cost)): raise Exception('Got Inf/NaN !') self.old_cost = new_cost self.update_params(self.lr) e_ed = time.time() msg = ( '.. iter %4d cost %.3g (before update %.3g), error %.3g step_size %.3g ' 'rho %.3g ' 'norm grad %.3g ' 'time [grad] %s,' '[updates param] %s,' 'whole time %s') print msg % (self.step, new_cost, old_cost, error, self.lr, rho, norm_grad, print_time(g_ed - g_st), print_time(e_ed - e_st), print_time(time.time() - self.step_timer)) self.step_timer = time.time() self.step += 1 ret = { 'cost': float(new_cost), 'error': float(error), 'time_grads': float(g_ed - g_st), 'time_eval': float(e_ed - e_st), 'norm_grad': norm_grad, 'lr': self.lr, 'rho': numpy.float32(rho) } return ret
def create_processed_vidstab_input(self, output): print("Create corrected 'rolling shutter' video for vidstab") crf = '16' ivid = path.join(self.cfg.frames_input_processed, '%06d.jpg') cmd = ['ffmpeg', '-framerate', self.cfg.fps, '-i', ivid, '-c:v', 'libx264', '-crf', crf, #'-vf', cropf, '-loglevel', 'error', '-stats', '-an', '-y', output] s = dt.datetime.now() run(cmd) e = dt.datetime.now() - s utils.print_time(e.total_seconds())
def login(user): """Log the user into a deployment of Nexus.""" config = config_utils.get_cli_config() c = config_utils.get_selected_deployment_config(config) if c is None: utils.error("You must select a deployment using `deployment --select` prior to running this command.") active_deployment_cfg = c[1] if user is None or user == '': user = input("Username:"******"Login successful")) if 'expires_in' in token: access_token = jwt.decode(token['access_token'], verify=False) expiry_utc = datetime.utcfromtimestamp(access_token['exp']) print("Token is valid for %s, expiry at %s" % (utils.print_time(token['expires_in']), utils.datetime_from_utc_to_local(expiry_utc))) active_deployment_cfg['token'] = token active_deployment_cfg['userinfo'] = userinfo config_utils.save_cli_config(config)
def validate(self): n_elems = 0 cost = 0 for batch in self.valid_data.__iter__(): n_elems += 1 cost += 100*self.model.validate(*batch) cost /= numpy.float32(n_elems) print ('** validation cost %6.3f computed in %s' ', best cost is %6.3f, test %6.3f, whole time %6.3f min') % ( cost, print_time(time.time() - self.batch_start_time), self.state['bvalidcost'], self.state['testcost'], (time.time() - self.start_time)/60. ) self.batch_start_time = time.time() pos = self.step // self.state['validFreq'] self.timings['valid'][pos] = float(cost) self.timings['test'][pos] = float(self.state['testcost']) self.state['validcost'] = float(cost) self.state['validtime'] = float(time.time() - self.start_time)/60. if self.state['bvalidcost'] > cost: self.state['bvalidcost'] = float(cost) self.state['btraincost'] = float(self.state['traincost']) self.test() print_mem('validate')
def print(self) -> str: s = max(self._bet_ref['finish_time'] - utils.now(), 0) time_remaining_str = utils.print_time(self._lang.get(), s) lines = [tr(self._lang.get(), 'BET.TIME', time=time_remaining_str)] if self._info_changed: self._stored_info = [] # Sort bet data total_bet = self.get_bet_sum() + self._bot.get_bet() bets = list(self._bet_ref['bets'].values()) bets.append((self._bot.icon, self._bot.get_bet())) bets.sort(key=lambda x: x[1], reverse=True) # Jackpot a: str = tr(self._lang.get(), 'BET.JACKPOT', EMOJI_SPARKLE=Emoji.SPARKLE, money=utils.print_money(self._lang.get(), total_bet)) b: str = tr(self._lang.get(), 'BET.MAX_BET', money=utils.print_money(self._lang.get(), self._limit)) self._stored_info.append(f"{a}\n{b}") # Player+bot bets for single_bet in bets: money_str = utils.print_money(self._lang.get(), single_bet[1]) pct = single_bet[1] / total_bet self._stored_info.append( f"{single_bet[0]}: {money_str} ({pct:.0%})") self._info_changed = False return '\n'.join(lines + self._stored_info)
def iter_batch(batch_queue, data_type, num): for i in range(num): while True: sleep_time = 0 # print('queue size: {}'.format(len(batch_queue))) # print('queue size: {}'.format(batch_queue.qsize())) while len(batch_queue) == 0: # while batch_queue.empty(): # tt = 10 if batch_queue.qsize()==0 else 1 tt = 10 time.sleep(tt) sleep_time += tt # print('sleep {} seconds.'.format(sleep_time)) # if sleep_time > 0: # print('sleep {} seconds.'.format(sleep_time)) bt = batch_queue.pop(0) # bt = batch_queue.get(block=False) if type(bt) != type('end'): # print('get one batch!') yield bt else: print('iter end one fp! {}'.format(utils.print_time())) name = multiprocessing.current_process().name print('{} current queue size: {}'.format(name, len(batch_queue))) break
def _print_epoch_info(self, accuracy, current_epoch, epochs, elapsed): epochs_past = self.get_past_epochs() - current_epoch accuracy_info = '' if accuracy is None else '| accuracy %d' % int( accuracy) epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past - 1) epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs, epoch_past_info) time_info = '%2dms/bt' % (elapsed / self.epoch_size * 1000) examples = int(np.floor(len(self.train_set) / FLAGS.batch_size)) loss_info = 't.loss:%d' % (self.epoch_stats.total_loss * 100 / (examples * np.prod(self.batch_shape[1:]))) info_string = ' '.join( [epoch_count, accuracy_info, time_info, loss_info]) ut.print_time(info_string, same_line=True)
def grid_search(train_data, test_data, y_train, y_test, model, seed=0, scorer=None, verbose=False, n_jobs=1, val_size=.1): ''' Grid Search the model. As the parameters are usually the same in a grid search, they are stored here already, easing the use. params: train_data: train data. Will be split into train and validation test_data: final test data y_train: labels for train data y_test: labels for the test data model: model to be grid searched ''' results = None # split the train and validation data X_train, X_val, y_train, y_val = cv.train_test_split(train_data, y_train, test_size=val_size, random_state=seed) # call the function according to model name model_name = model.__class__.__name__ if model_name in ('GradientBoostingClassifier'): param_grid = {'learning_rate': [0.1, 0.3], 'n_estimators': [200], 'max_depth': [3, 9, 18], 'subsample': [1], 'min_samples_leaf': [1, 7, 14, 20], 'max_features': [.4, .9, None], } # grid search Gradient Boosting print_time('Starting grid search') gbr_cv = GradientBoostingClassifier() gs_cv = GridSearchCV(gbr_cv, param_grid, verbose=verbose, n_jobs=n_jobs, scoring=make_scorer(scorer)).\ fit(X_train, np.ravel(y_train)) gbr = gs_cv.best_estimator_ # validation dataset pred_val = gbr.predict(X_val) score_val = scorer(y_val, pred_val) print_time('Validation prediction score: {}'.format(score_val)) # check the performance against the test data pred_test = gbr.predict(test_data) score_test = scorer(y_test, pred_test) print_time('Test prediction score: {}'.format(score_test)) results = {'search': gs_cv, 'pred_test': pred_test, 'score_test': score_test, 'pred_val': pred_val, 'score_val': score_val} print_time('Finished grid search') return results
def text8(min_freq=5, max_vocab_size=None): """Text8 dataset helper. Parameters ---------- min_freq : int, default 5 Minimum token frequency for a token to be included in the vocabulary and returned DataStream. max_vocab_size : int, optional Specifies a maximum size for the vocabulary. Returns ------- gluonnlp.data.DataStream Each sample is a valid input to gluonnlp.data.EmbeddingCenterContextBatchify. gluonnlp.Vocab Vocabulary of all tokens in Text8 that occur at least min_freq times of maximum size max_vocab_size. idx_to_counts : list of int Mapping from token indices to their occurrence-counts in the Text8 dataset. """ with print_time('read data'): data = nlp.data.Text8(segment='train') counter = nlp.data.count_tokens(itertools.chain.from_iterable(data)) vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None, bos_token=None, eos_token=None, min_freq=min_freq, max_size=max_vocab_size) idx_to_counts = [counter[w] for w in vocab.idx_to_token] def code(sentence): return [vocab[token] for token in sentence if token in vocab] with print_time('code data'): data = data.transform(code, lazy=False) data = nlp.data.SimpleDataStream([data]) return data, vocab, idx_to_counts
def train(self, epochs_to_train=5): meta = self.get_meta() ut.print_time('train started: \n%s' % ut.to_file_name(meta)) # return meta, np.random.randn(epochs_to_train) ut.configure_folders(FLAGS, meta) self._dataset, self._filters = self.fetch_datasets(self._activation) self.build_model() self._register_training_start() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) self._saver = tf.train.Saver() if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()): self._saver.restore(sess, self.get_checkpoint_path()) ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31) # MAIN LOOP for current_epoch in xrange(epochs_to_train): feed, permutation = self._get_epoch_dataset() for _, batch in enumerate(feed): filter = batch[1][0] assert batch[1][0,0] == batch[1][-1,0] encoding, = sess.run([self._encode], feed_dict={self._input: batch[0]}) # 1.1 encode forward clamped_enc, vae_grad = _clamp(encoding, filter) # 1.2 # clamp sess.run(self._assign_clamped, feed_dict={self._clamped:clamped_enc}) reconstruction, loss, clamped_gradient, _ = sess.run( # 2.1 decode forward+backward [self._decode, self._decoder_loss, self._clamped_grad, self._train_decoder], feed_dict={self._clamped: clamped_enc, self._reconstruction: batch[0]}) declamped_grad = _declamp_grad(vae_grad, clamped_gradient, filter) # 2.2 prepare gradient _, step = sess.run( # 3.0 encode backward path [self._train_encoder, self._step], feed_dict={self._input: batch[0], self._encoding: encoding-declamped_grad}) # Profit self._register_batch(batch, encoding, reconstruction, loss) self._register_epoch(current_epoch, epochs_to_train, permutation, sess) self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph) meta = self._register_training() return meta, self._stats['epoch_accuracy']
def preprocess_dataset(data, min_freq=5, max_vocab_size=None): """Dataset preprocessing helper. Parameters ---------- data : mx.data.Dataset Input Dataset. For example gluonnlp.data.Text8 or gluonnlp.data.Fil9 min_freq : int, default 5 Minimum token frequency for a token to be included in the vocabulary and returned DataStream. max_vocab_size : int, optional Specifies a maximum size for the vocabulary. Returns ------- gluonnlp.data.DataStream Each sample is a valid input to gluonnlp.data.EmbeddingCenterContextBatchify. gluonnlp.Vocab Vocabulary of all tokens in Text8 that occur at least min_freq times of maximum size max_vocab_size. idx_to_counts : list of int Mapping from token indices to their occurrence-counts in the Text8 dataset. """ with print_time('count and construct vocabulary'): counter = nlp.data.count_tokens(itertools.chain.from_iterable(data)) vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None, bos_token=None, eos_token=None, min_freq=min_freq, max_size=max_vocab_size) idx_to_counts = [counter[w] for w in vocab.idx_to_token] def code(sentence): return [vocab[token] for token in sentence if token in vocab] with print_time('code data'): data = data.transform(code, lazy=False) data = nlp.data.SimpleDataStream([data]) return data, vocab, idx_to_counts
def train_regression(data, target, models=None, scorer=None, verbose=False, seed=None): ''' Helper to split the models always the same :return: list of tuples (model id, dict of model) :params: data: data to be trained, no target values target: model target val models: models to be trained, inside a list [] scorer: the scorer to order the best results verbose: print information ''' seed = seed if seed else get_seed() # get the models to test if not models: models = [ExtraTreesClassifier(random_state=seed), GradientBoostingClassifier(random_state=seed), RandomForestClassifier(random_state=seed), LogisticRegressionCV(), RidgeClassifierCV(), LinearSVC(random_state=seed), SVC(random_state=seed), SGDClassifier(random_state=seed), GaussianNB()] # choose accuracy if no scorer was passed if not scorer: scorer = accuracy_score # split in train and validation set X_train, X_val, y_train, y_val = split_data(data, target, seed) print_time('Created train and validation') print_time('Size train: {} test:{}'. format(X_train.shape, X_val.shape)) return train_all_models(X_train, X_val, y_train, y_val, models, scorer, verbose)
def validate(self): cost = self.model.validate() print ('** validation cost %6.3f computed in %s' ', best cost is %6.3f, test %6.3f, whole time %6.3f min') % ( cost, print_time(time.time() - self.batch_start_time), self.state['bvalidcost'], self.state['testcost'], (time.time() - self.start_time)/60. ) self.batch_start_time = time.time() pos = self.step // self.state['validFreq'] self.timings['valid'][pos] = float(cost) self.timings['test'][pos] = float(self.state['testcost']) self.state['validcost'] = float(cost) self.state['validtime'] = float(time.time() - self.start_time)/60. if self.state['bvalidcost'] > cost: self.state['bvalidcost'] = float(cost) self.state['btraincost'] = float(self.state['traincost']) self.test() print_mem('validate')
def th_loadfile(batch_queue, fp_list): name = multiprocessing.current_process().name for fp in fp_list: while len(batch_queue) > 100: # while batch_queue.full(): time.sleep(5) # print('{} sleep for 60 seconds.'.format(name)) bts = np.load(fp) # print('{} originally has {} batches.'.format(fp, len(bts))) np.random.shuffle(bts) # print('{} has {} batches to put.'.format(fp, len(bts))) for bt in bts: batch_queue.append(bt) # batch_queue.put(bt, block=True) batch_queue.append('end') # batch_queue.put('end', block=True) print('{} finished {}. {}'.format(name, fp, utils.print_time())) print('{} current queue size: {}'.format(name, len(batch_queue))) print('{} ends.'.format(name))
def main(self): print_mem('start') self.state['gotNaN'] = 0 self.start_time = time.time() self.batch_start_time = time.time() self.step = 0 self.save_iter = 0 self.save() if self.channel is not None: self.channel.save() self.save_time = time.time() last_cost = 1. start_time = time.time() self.start_time = start_time while self.step < self.state['loopIters'] and \ last_cost > .1*self.state['minerr'] and \ (time.time() - start_time)/60. < self.state['timeStop']: if (time.time() - self.save_time)/60. > self.state['saveFreq']: self.save() if self.channel is not None: self.channel.save() self.save_time = time.time() st = time.time() try: rvals = self.algo() self.state['traincost'] = float(rvals['cost']) self.state['step'] = self.step last_cost = rvals['cost'] for name in rvals.keys(): pos = self.step // self.state['trainFreq'] self.timings[name][pos] = rvals[name] if numpy.isinf(rvals['cost']) or numpy.isnan(rvals['cost']): self.state['gotNaN'] = 1 self.save() if self.channel: self.channel.save() print 'Got NaN while training' last_cost = 0 if self.step % self.state['validFreq'] == 0: self.validate() self.step += 1 except: self.state['wholetime'] = float(time.time() - start_time) self.save() if self.channel: self.channel.save() last_cost = 0 print 'Error in running natgrad (lr issue)' print 'BEST SCORE' print 'Validation', self.state['validcost'] print 'Validation time', print_time(self.state['validtime']) print 'Train cost', self.state['traincost'] print 'Best Train', self.state['btraincost'] print 'Best Valid', self.state['bvalidcost'] print 'TEST', self.state['testcost'] print 'Took', (time.time() - start_time)/60.,'min' raise self.state['wholetime'] = float(time.time() - start_time) self.validate() self.save() if self.channel: self.channel.save() print 'BEST SCORE' print 'Validation', self.state['validcost'] print 'Validation time', print_time(self.state['validtime']) print 'Train cost', self.state['traincost'] print 'Best Train', self.state['btraincost'] print 'Best Valid', self.state['bvalidcost'] print 'TEST', self.state['testcost'] print 'Took', (time.time() - start_time)/60.,'min'
def visualize_encodings(encodings, file_name=None, grid=None, skip_every=999, fast=False, fig=None, interactive=False): # encodings = encodings[0:360] if len(encodings) < 1500 else encodings[0:720] encodings = manual_pca(encodings) if encodings.shape[1] <= 3: return print_data_only(encodings, file_name, fig=fig, interactive=interactive) encodings = encodings[0:720] hessian_euc = dist.squareform(dist.pdist(encodings[0:720], 'euclidean')) hessian_cos = dist.squareform(dist.pdist(encodings[0:720], 'cosine')) grid = (3, 4) if grid is None else grid project_ops = [] n = 2 project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa'))) project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified'))) project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) project_ops.append(("TSNE 30/2000 N:%d" % n, TSNE(perplexity=30, n_components=n, init='pca',n_iter=2000))) n = 3 project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa'))) project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified'))) project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) project_ops.append(('MDS cosine N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed'))) # print( # np.min(hessian_euc), # np.min(hessian_cos), # hessian_euc.size - np.count_nonzero(hessian_euc)) plot_places = [] for i in range(12): u, v = int(i/(skip_every-1)), i % (skip_every - 1) j = v + u * skip_every + 1 plot_places.append(j) fig = fig if fig is not None else plt.figure() fig.set_size_inches(fig.get_size_inches()[0] * grid[0] /1., fig.get_size_inches()[1] * grid[1]/2.0) for i, (name, manifold) in enumerate(project_ops): is3d = 'N:3' in name # if (fast or 'grid' in FLAGS.suffix) and 'MDS' in name: # continue try: if is3d: subplot = plt.subplot(grid[0], grid[1], plot_places[i], projection='3d') else: subplot = plt.subplot(grid[0], grid[1], plot_places[i]) data_source = encodings if not _needs_hessian(manifold) else \ (hessian_cos if 'cosine' in name else hessian_euc) projections = manifold.fit_transform(data_source) scatter(subplot, projections, is3d, build_radial_colors(len(data_source))) subplot.set_title(name) except: print(name, "Unexpected error: ", sys.exc_info()[0], sys.exc_info()[1] if len(sys.exc_info()) > 1 else '') visualize_data_same(encodings, grid=grid, places=plot_places[-4:]) # visualize_data_same(encodings, grid=grid, places=np.arange(13, 17), dims_as_colors=True) # fig.tight_layout() if not interactive: save_fig(file_name, fig) ut.print_time('visualization finished')
def train(args): """Training helper.""" if not args.model.lower() in ['cbow', 'skipgram']: logging.error('Unsupported model %s.', args.model) sys.exit(1) if args.data.lower() == 'toy': data = mx.gluon.data.SimpleDataset(nlp.data.Text8(segment='train')[:2]) data, vocab, idx_to_counts = preprocess_dataset( data, max_vocab_size=args.max_vocab_size) elif args.data.lower() == 'text8': data = nlp.data.Text8(segment='train') data, vocab, idx_to_counts = preprocess_dataset( data, max_vocab_size=args.max_vocab_size) elif args.data.lower() == 'fil9': data = nlp.data.Fil9(max_sentence_length=10000) data, vocab, idx_to_counts = preprocess_dataset( data, max_vocab_size=args.max_vocab_size) elif args.data.lower() == 'wiki': data, vocab, idx_to_counts = wiki(args.wiki_root, args.wiki_date, args.wiki_language, args.max_vocab_size) if args.ngram_buckets > 0: data, batchify_fn, subword_function = transform_data_fasttext( data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow', ngram_buckets=args.ngram_buckets, ngrams=args.ngrams, batch_size=args.batch_size, window_size=args.window, frequent_token_subsampling=args.frequent_token_subsampling) else: subword_function = None data, batchify_fn = transform_data_word2vec( data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow', batch_size=args.batch_size, window_size=args.window, frequent_token_subsampling=args.frequent_token_subsampling) num_tokens = float(sum(idx_to_counts)) model = CBOW if args.model.lower() == 'cbow' else SG embedding = model(token_to_idx=vocab.token_to_idx, output_dim=args.emsize, batch_size=args.batch_size, num_negatives=args.negative, negatives_weights=mx.nd.array(idx_to_counts), subword_function=subword_function) context = get_context(args) embedding.initialize(ctx=context) if not args.no_hybridize: embedding.hybridize(static_alloc=True, static_shape=True) optimizer_kwargs = dict(learning_rate=args.lr) try: trainer = mx.gluon.Trainer(embedding.collect_params(), args.optimizer, optimizer_kwargs) except ValueError as e: if args.optimizer == 'groupadagrad': logging.warning('MXNet <= v1.3 does not contain ' 'GroupAdaGrad support. Falling back to AdaGrad') trainer = mx.gluon.Trainer(embedding.collect_params(), 'adagrad', optimizer_kwargs) else: raise e try: if args.no_prefetch_batch: data = data.transform(batchify_fn) else: from executors import LazyThreadPoolExecutor num_cpu = len(os.sched_getaffinity(0)) ex = LazyThreadPoolExecutor(num_cpu) except (ImportError, SyntaxError, AttributeError): # Py2 - no async prefetching is supported logging.warning( 'Asynchronous batch prefetching is not supported on Python 2. ' 'Consider upgrading to Python 3 for improved performance.') data = data.transform(batchify_fn) num_update = 0 prefetched_iters = [] for _ in range(min(args.num_prefetch_epoch, args.epochs)): prefetched_iters.append(iter(data)) for epoch in range(args.epochs): if epoch + len(prefetched_iters) < args.epochs: prefetched_iters.append(iter(data)) data_iter = prefetched_iters.pop(0) try: batches = ex.map(batchify_fn, data_iter) except NameError: # Py 2 or batch prefetching disabled batches = data_iter # Logging variables log_wc = 0 log_start_time = time.time() log_avg_loss = 0 for i, batch in enumerate(batches): ctx = context[i % len(context)] batch = [array.as_in_context(ctx) for array in batch] with mx.autograd.record(): loss = embedding(*batch) loss.backward() num_update += loss.shape[0] if len(context) == 1 or (i + 1) % len(context) == 0: trainer.step(batch_size=1) # Logging log_wc += loss.shape[0] log_avg_loss += loss.mean().as_in_context(context[0]) if (i + 1) % args.log_interval == 0: # Forces waiting for computation by computing loss value log_avg_loss = log_avg_loss.asscalar() / args.log_interval wps = log_wc / (time.time() - log_start_time) # Due to subsampling, the overall number of batches is an upper # bound num_batches = num_tokens // args.batch_size if args.model.lower() == 'skipgram': num_batches = (num_tokens * args.window * 2) // args.batch_size else: num_batches = num_tokens // args.batch_size logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ' 'throughput={:.2f}K wps, wc={:.2f}K'.format( epoch, i + 1, num_batches, log_avg_loss, wps / 1000, log_wc / 1000)) log_start_time = time.time() log_avg_loss = 0 log_wc = 0 if args.eval_interval and (i + 1) % args.eval_interval == 0: with print_time('mx.nd.waitall()'): mx.nd.waitall() with print_time('evaluate'): evaluate(args, embedding, vocab, num_update) # Evaluate with print_time('mx.nd.waitall()'): mx.nd.waitall() with print_time('evaluate'): evaluate(args, embedding, vocab, num_update, eval_analogy=not args.no_eval_analogy) # Save params with print_time('save parameters'): embedding.save_parameters(os.path.join(args.logdir, 'embedding.params'))
def __call__(self): self.data.update_before_computing_gradients() g_st = time.time() self.compute_gradients() g_ed = time.time() self.data.update_before_computing_natural_gradients() r_st = time.time() rvals = self.compute_natural_gradients() r_ed = time.time() self.data.update_before_evaluation() e_st = time.time() old_cost = self.compute_old_cost() new_cost, error = self.compute_new_cost(self.lr) rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr, rvals[5]*rvals[6]) if self.state['adapt'] == 3: odamp = self.damping.get_value() if rvals[1] < 5 and odamp > 1e-5: self.damping.set_value(odamp * 2. /3.) if rvals[1] > 30 and odamp < 1024: self.damping.set_value(odamp * 3/ 2.) if self.state['adapt'] == 1: if rho < .25 and self.damping.get_value() > 0: self.damping.set_value(numpy.float32( self.damping.get_value() * 3. / 2.)) elif rho < .25: self.damping.set_value(numpy.float32(1e-5)) elif rho > .75 and self.damping.get_value() > 1e-4: self.damping.set_value(numpy.float32( self.damping.get_value() * 2. / 3.)) #elif rho > .75: # self.damping.set_value(numpy.float32(0.)) tmp_cost = new_cost if self.state['adapt'] == 4: if self.step > self.state['adapt_start']: if self.step % self.state['adapt_change'] == 0: self.damping.set_value( self.damping.get_value() * self.state['adapt_decrease']) if self.state['lr_adapt'] == 1: if self.step > self.state['lr_adapt_start']: if self.step % self.state['lr_adapt_change'] == 0: self.lr = self.lr * self.state['lr_adapt_decrease'] elif self.state['lr_adapt'] == 2: if self.step > self.state['lr_adapt_start']: self.lr = self.state['lr0'] /\ (1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta']) self.state['lr'] = float(self.lr) if self.step % self.state['trainFreq'] == 0: new_cost, old_cost, error = self.compute_new_cost_all(self.lr) if new_cost > self.state['btraincost'] * 6: raise Exception('Variance too large on training cost!') if self.state['adapt'] == 2: rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr, rvals[5]*rvals[6]) if rho < .25 and self.damping.get_value() > 0: self.damping.set_value(numpy.float32( self.damping.get_value() * 3. / 2.)) elif rho < .25: self.damping.set_value(numpy.float32(1e-6)) elif rho > .75 and self.damping.get_value() > 1e-6: self.damping.set_value(numpy.float32( self.damping.get_value() * 2. / 3.)) elif rho > .75: self.damping.set_value(numpy.float32(0.)) self.__new_cost = new_cost self.__error = error e_ed = time.time() print 'Minres: %s' % self.msgs[rvals[0]], \ '# iters %04d' % rvals[1], \ 'relative error residuals %.4g' % rvals[2], \ 'Anorm', rvals[3], 'Acond', rvals[4] msg = ('.. iter %4d cost %.3g (%.3g), error %.3g step_size %.3g ' 'rho %.3g damping %.4g ' 'r_g %.3g ' 'ord0_norm %.3g ' 'norm grad %.3g ' 'norm nat grad %.3g ' 'angle %.3g ' 'time [grad] %s,' '[riemann grad] %s,' '[updates param] %s,' 'whole time %s') print msg % ( self.step, new_cost, tmp_cost, error, self.lr, rho, self.damping.get_value(), r_g, rvals[7], rvals[5], rvals[6], angle, print_time(g_ed - g_st), print_time(r_ed - r_st), print_time(e_ed - e_st), print_time(time.time() - self.step_timer)) self.step_timer = time.time() else: new_cost = self.__new_cost error = self.__error self.old_cost = new_cost self.update_params(self.lr) e_ed = time.time() self.step += 1 ret = { 'cost': float(new_cost), 'error': float(error), 'time_grads': float(g_ed - g_st), 'time_metric': float(r_ed - r_st), 'time_eval': float(e_ed - e_st), 'minres_flag': rvals[0], 'minres_iters': rvals[1], 'minres_relres': rvals[2], 'minres_Anorm': rvals[3], 'minres_Acond': rvals[4], 'norm_ord0': rvals[7], 'norm_grad':rvals[5], 'norm_nat': rvals[6], 'grad_angle' : float(angle), 'lr': self.lr, 'r_g': float(r_g), 'icost' : float(tmp_cost), 'damping': self.damping.get_value(), 'rho': numpy.float32(rho)} return ret
def new_iter_batch(qlts, num, data_type): n = 0 n_cores = len(qlts) sleep_t = 0 cache_key_list = cache_data_dict[data_type].keys() random.shuffle(cache_key_list) while n < num: empty = 0 for qlt in qlts: batch_queue, lock, i = qlt if len(batch_queue) == 0: empty += 1 continue if lock.acquire(False): print('[Main th] Slept for {} secs.'.format(sleep_t)) # sleep_t = 0 # print('Taking batches from batch_queue {}'.format(i)) # bts = [bt for bt in batch_queue] # batch_queue.clear() # del batch_queue[:] bts = [] while len(batch_queue) > 0: bts.append(batch_queue.popleft()) print('[Main th] Got {} batches from Queue {}.\t{}'.format(len(bts), i, utils.print_time())) lock.release() lg = get_batch_length(bts[0]) st = time.time() n_elem = 0 for bt in bts: if type(bt) != type('end'): # ti = time.time() yield bt n_elem += 1 else: n += 1 print('[Main th] Finished {}: {}*{}.\t{:.3f} secs.'.format(n, lg, n_elem, time.time()-st)) n_elem = 0 sys.stdout.flush() del bts[:] ng = gc.collect() # print('{} garbages.'.format(ng)) del gc.garbage[:] else: empty += 1 if len(cache_key_list) > 0: empty = 0 dt_n = cache_key_list.pop() bts = cache_data_dict[data_type][dt_n] print('[Main th] Got {} batches from Cache {}_{}.\t{}'.format(len(bts), data_type, dt_n, utils.print_time())) random.shuffle(bts) lg = get_batch_length(bts[0]) st = time.time() n_elem = 0 for bt in bts: yield bt n_elem += 1 n += 1 print('[Main th] Finished {}: {}*{}.\t{:.3f} secs.'.format(n, lg, n_elem, time.time()-st)) n_elem = 0 sys.stdout.flush() if empty == n_cores: # print('All empty or unavailable.') time.sleep(2) sleep_t += 2 print('[Main th] Totally slept {} secs.'.format(sleep_t))
def transform_data_fasttext(data, vocab, idx_to_counts, cbow, ngram_buckets, ngrams, batch_size, window_size, frequent_token_subsampling=1E-4, dtype='float32', index_dtype='int64'): """Transform a DataStream of coded DataSets to a DataStream of batches. Parameters ---------- data : gluonnlp.data.DataStream DataStream where each sample is a valid input to gluonnlp.data.EmbeddingCenterContextBatchify. vocab : gluonnlp.Vocab Vocabulary containing all tokens whose indices occur in data. For each token, it's associated subwords will be computed and used for constructing the batches. No subwords are used if ngram_buckets is 0. idx_to_counts : list of int List of integers such that idx_to_counts[idx] represents the count of vocab.idx_to_token[idx] in the underlying dataset. The count information is used to subsample frequent words in the dataset. Each token is independently dropped with probability 1 - sqrt(t / (count / sum_counts)) where t is the hyperparameter frequent_token_subsampling. cbow : boolean If True, batches for CBOW are returned. ngram_buckets : int Number of hash buckets to consider for the fastText nlp.vocab.NGramHashes subword function. ngrams : list of int For each integer n in the list, all ngrams of length n will be considered by the nlp.vocab.NGramHashes subword function. batch_size : int The returned data stream iterates over batches of batch_size. window_size : int The context window size for gluonnlp.data.EmbeddingCenterContextBatchify. frequent_token_subsampling : float Hyperparameter for subsampling. See idx_to_counts above for more information. dtype : str or np.dtype, default 'float32' Data type of data array. index_dtype : str or np.dtype, default 'int64' Data type of index arrays. Returns ------- gluonnlp.data.DataStream Stream over batches. Each returned element is a list corresponding to the arguments for the forward pass of model.SG or model.CBOW respectively based on if cbow is False or True. If ngarm_buckets > 0, the returned sample will contain ngrams. Both model.SG or model.CBOW will handle them correctly as long as they are initialized with the subword_function returned as second argument by this function (see below). gluonnlp.vocab.NGramHashes The subword_function used for obtaining the subwords in the returned batches. """ if ngram_buckets <= 0: raise ValueError('Invalid ngram_buckets. Use Word2Vec training ' 'pipeline if not interested in ngrams.') sum_counts = float(sum(idx_to_counts)) idx_to_pdiscard = [ 1 - math.sqrt(frequent_token_subsampling / (count / sum_counts)) for count in idx_to_counts] def subsample(shard): return [[ t for t, r in zip(sentence, np.random.uniform(0, 1, size=len(sentence))) if r > idx_to_pdiscard[t]] for sentence in shard] data = data.transform(subsample) batchify = nlp.data.batchify.EmbeddingCenterContextBatchify( batch_size=batch_size, window_size=window_size, cbow=cbow, weight_dtype=dtype, index_dtype=index_dtype) data = data.transform(batchify) with print_time('prepare subwords'): subword_function = nlp.vocab.create_subword_function( 'NGramHashes', ngrams=ngrams, num_subwords=ngram_buckets) # Store subword indices for all words in vocabulary idx_to_subwordidxs = list(subword_function(vocab.idx_to_token)) subwordidxs = np.concatenate(idx_to_subwordidxs) subwordidxsptr = np.cumsum([ len(subwordidxs) for subwordidxs in idx_to_subwordidxs]) subwordidxsptr = np.concatenate([ np.zeros(1, dtype=np.int64), subwordidxsptr]) if cbow: subword_lookup = functools.partial( cbow_lookup, subwordidxs=subwordidxs, subwordidxsptr=subwordidxsptr, offset=len(vocab)) else: subword_lookup = functools.partial( skipgram_lookup, subwordidxs=subwordidxs, subwordidxsptr=subwordidxsptr, offset=len(vocab)) max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs) if max_subwordidxs_len > 500: warnings.warn( 'The word with largest number of subwords ' 'has {} subwords, suggesting there are ' 'some noisy words in your vocabulary. ' 'You should filter out very long words ' 'to avoid memory issues.'.format(max_subwordidxs_len)) data = UnchainStream(data) if cbow: batchify_fn = cbow_fasttext_batch else: batchify_fn = skipgram_fasttext_batch batchify_fn = functools.partial( batchify_fn, num_tokens=len(vocab) + len(subword_function), subword_lookup=subword_lookup, dtype=dtype, index_dtype=index_dtype) return data, batchify_fn, subword_function
def main(_=None, weight_init=tf.random_normal, activation_f=tf.nn.sigmoid, data_min=0, data_scale=1.0, epochs=50, learning_rate=0.01, prefix=None): tf.reset_default_graph() input_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1]) output_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1]) # Grab the data as numpy arrays. train_input, train_output = data_utils.mnist(training=True) test_input, test_output = data_utils.mnist(training=False) train_set = ut.mnist_select_n_classes(train_input, train_output, NUM_CLASSES, min=data_min, scale=data_scale) test_set = ut.mnist_select_n_classes(test_input, test_output, NUM_CLASSES, min=data_min, scale=data_scale) train_input, train_output = train_set[0], train_set[0] test_input, test_output = test_set[0], test_set[0] ut.print_info('train (min, max): (%f, %f)' % (np.min(train_set[0]), np.max(train_set[0]))) visual_inputs, visual_output = train_set[0][0:BATCH_SIZE], train_set[0][0:BATCH_SIZE] epoch_reconstruction = [] EPOCH_SIZE = len(train_input) // BATCH_SIZE TEST_SIZE = len(test_input) // BATCH_SIZE assert_model(input_placeholder, output_placeholder, test_input, test_output, train_input, train_output, visual_inputs, visual_output) with pt.defaults_scope(activation_fn=activation_f, # batch_normalize=True, # learned_moments_update_rate=0.0003, # variance_epsilon=0.001, # scale_after_normalization=True ): with pt.defaults_scope(phase=pt.Phase.train): with tf.variable_scope("model") as scope: output_tensor = decoder(encoder(input_placeholder), weight_init=weight_init) pretty_loss = loss(output_tensor, output_placeholder) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) train = pt.apply_optimizer(optimizer, losses=[pretty_loss]) init = tf.initialize_all_variables() runner = pt.train.Runner(save_path=FLAGS.save_path) best_q = 100000 with tf.Session() as sess: sess.run(init) for epoch in xrange(epochs): # Shuffle the training data. additional_info = '' if epoch % np.ceil(epochs / 40.0) == 0 or epoch + 1 == epochs: reconstruct, loss_value = sess.run([output_tensor, pretty_loss], {input_placeholder: visual_inputs, output_placeholder: visual_output}) epoch_reconstruction.append(reconstruct) additional_info += 'epoch:%d (min, max): (%f %f)' %(epoch, np.min(reconstruct), np.max(reconstruct)) train_input, train_output = data_utils.permute_data( (train_input, train_output)) runner.train_model( train, pretty_loss, EPOCH_SIZE, feed_vars=(input_placeholder, output_placeholder), feed_data=pt.train.feed_numpy(BATCH_SIZE, train_input, train_output), print_every=None ) accuracy = runner.evaluate_model( pretty_loss, TEST_SIZE, feed_vars=(input_placeholder, output_placeholder), feed_data=pt.train.feed_numpy(BATCH_SIZE, test_input, test_output)) ut.print_time('Accuracy after %2d/%d epoch %.2f; %s' % (epoch + 1, epochs, accuracy, additional_info)) if best_q > accuracy: best_q = accuracy save_params = {'suf': 'mn_basic', 'act': activation_f, 'e': epochs, 'opt': optimizer, 'lr': learning_rate, 'init': weight_init, 'acu': int(best_q), 'bs': BATCH_SIZE, 'h': HIDDEN_0_SIZE, 'i':prefix} ut.reconstruct_images_epochs(np.asarray(epoch_reconstruction), visual_output, save_params=save_params) ut.print_time('Best Quality: %f for %s' % (best_q, ut.to_file_name(save_params))) ut.reset_start_time() return best_q
def process_batches(batch_queue, network, exec_func, func): print('[Queue {}] Start {} {} batches.\t{}'.format(idx, dt_type, len(batch_queue), utils.print_time())) lg = get_batch_length(batch_queue[0]) st = time.time() n_elem = 0 n = shared_param.finished_num if hasattr(shared_param, 'model'): lasagne.layers.set_all_param_values(network, shared_param.model) for bt in batch_queue: if type(bt) != type('end'): exec_func(bt, func, shared_param) n_elem += 1 else: n += 1 print('[Queue {}] Finished {}: {}*{}.\t{:.3f} secs.'.format(idx, n, lg, n_elem, time.time()-st)) n_elem = 0 sys.stdout.flush() batch_queue.clear() shared_param.finished_num = n shared_param.model = lasagne.layers.get_all_param_values(network)
def main(_=None, weight_init=None, activation_f=tf.nn.sigmoid, data_min=0, data_scale=1.0, epochs=3,learning_rate=None): tf.reset_default_graph() input_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 2]) output_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1]) # Grab the data as numpy arrays. train_input, train_output = data_utils.mnist(training=True) test_input, test_output = data_utils.mnist(training=False) train_set = ut.mnist_select_n_classes(train_input, train_output, NUM_CLASSES, min=data_min, scale=data_scale) test_set = ut.mnist_select_n_classes(test_input, test_output, NUM_CLASSES, min=data_min, scale=data_scale) train_input, train_output = train_set[1], train_set[0] test_input, test_output = test_set[1], test_set[0] ut.print_info('train (min, max): (%f, %f)' % (np.min(train_set[0]), np.max(train_set[0]))) visual_inputs, visual_output = train_set[1][0:BATCH_SIZE], train_set[0][0:BATCH_SIZE] epoch_reconstruction = [] EPOCH_SIZE = len(train_input) // BATCH_SIZE TEST_SIZE = len(test_input) // BATCH_SIZE ut.print_info('train: %s' % str(train_input.shape)) ut.print_info('test: %s' % str(test_input.shape)) ut.print_info('output shape: %s' % str(train_output[0].shape)) assert visual_inputs.shape == input_placeholder.get_shape() assert len(train_input.shape) == len(input_placeholder.get_shape()) assert len(test_input.shape) == len(input_placeholder.get_shape()) assert visual_output.shape == output_placeholder.get_shape() assert len(train_output.shape) == len(output_placeholder.get_shape()) assert len(test_output.shape) == len(output_placeholder.get_shape()) with pt.defaults_scope(activation_fn=activation_f, # batch_normalize=True, # learned_moments_update_rate=0.0003, # variance_epsilon=0.001, # scale_after_normalization=True ): with pt.defaults_scope(phase=pt.Phase.train): with tf.variable_scope("model") as scope: output_tensor = decoder(encoder(input_placeholder), weight_init=weight_init) pretty_loss = loss(output_tensor, output_placeholder) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) # optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) train = pt.apply_optimizer(optimizer, losses=[pretty_loss]) init = tf.initialize_all_variables() runner = pt.train.Runner(save_path=FLAGS.save_path) best_q = 100000 with tf.Session() as sess: sess.run(init) for epoch in xrange(epochs): # Shuffle the training data. if epoch % np.ceil(epochs / 40.0) == 0 or epoch + 1 == epochs: reconstruct, loss_value = sess.run([output_tensor, pretty_loss], {input_placeholder: visual_inputs, output_placeholder: visual_output}) epoch_reconstruction.append(reconstruct) ut.print_info('epoch:%d (min, max): (%f %f)' %(epoch, np.min(reconstruct), np.max(reconstruct))) train_input, train_output = data_utils.permute_data( (train_input, train_output)) runner.train_model( train, pretty_loss, EPOCH_SIZE, feed_vars=(input_placeholder, output_placeholder), feed_data=pt.train.feed_numpy(BATCH_SIZE, train_input, train_output) ) accuracy = runner.evaluate_model( pretty_loss, TEST_SIZE, feed_vars=(input_placeholder, output_placeholder), feed_data=pt.train.feed_numpy(BATCH_SIZE, test_input, test_output)) ut.print_time('Accuracy after %d epoch %g%%' % ( epoch + 1, accuracy * 100)) if best_q > accuracy * 10: best_q = accuracy * 10 ut.reconstruct_images_epochs(np.asarray(epoch_reconstruction), visual_output, save_params={'suf':'mn_trivs', 'act':activation_f, 'e':epochs, 'opt':optimizer, 'lr': learning_rate, 'init':weight_init, 'acu': int(best_q)})
def train(args): """Training helper.""" vocab, row, col, counts = get_train_data(args) model = GloVe(token_to_idx=vocab.token_to_idx, output_dim=args.emsize, dropout=args.dropout, x_max=args.x_max, alpha=args.alpha, weight_initializer=mx.init.Uniform(scale=1 / args.emsize)) context = get_context(args) model.initialize(ctx=context) if not args.no_hybridize: model.hybridize(static_alloc=not args.no_static_alloc) optimizer_kwargs = dict(learning_rate=args.lr, eps=args.adagrad_eps) params = list(model.collect_params().values()) try: trainer = mx.gluon.Trainer(params, 'groupadagrad', optimizer_kwargs) except ValueError: logging.warning('MXNet <= v1.3 does not contain ' 'GroupAdaGrad support. Falling back to AdaGrad') trainer = mx.gluon.Trainer(params, 'adagrad', optimizer_kwargs) index_dtype = 'int32' if counts.shape[0] >= np.iinfo(np.int32).max: index_dtype = 'int64' logging.info('Co-occurrence matrix is large. ' 'Using int64 to represent sample indices.') indices = mx.nd.arange(counts.shape[0], dtype=index_dtype) for epoch in range(args.epochs): # Logging variables log_wc = 0 log_start_time = time.time() log_avg_loss = 0 mx.nd.shuffle(indices, indices) # inplace shuffle bs = args.batch_size num_batches = indices.shape[0] // bs for i in range(num_batches): batch_indices = indices[bs * i:bs * (i + 1)] ctx = context[i % len(context)] batch_row = row[batch_indices].as_in_context(ctx) batch_col = col[batch_indices].as_in_context(ctx) batch_counts = counts[batch_indices].as_in_context(ctx) with mx.autograd.record(): loss = model(batch_row, batch_col, batch_counts) loss.backward() if len(context) == 1 or (i + 1) % len(context) == 0: trainer.step(batch_size=1) # Logging log_wc += loss.shape[0] log_avg_loss += loss.mean().as_in_context(context[0]) if (i + 1) % args.log_interval == 0: # Forces waiting for computation by computing loss value log_avg_loss = log_avg_loss.asscalar() / args.log_interval wps = log_wc / (time.time() - log_start_time) logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ' 'throughput={:.2f}K wps, wc={:.2f}K'.format( epoch, i + 1, num_batches, log_avg_loss, wps / 1000, log_wc / 1000)) log_dict = dict( global_step=epoch * len(indices) + i * args.batch_size, epoch=epoch, batch=i + 1, loss=log_avg_loss, wps=wps / 1000) log(args, log_dict) log_start_time = time.time() log_avg_loss = 0 log_wc = 0 if args.eval_interval and (i + 1) % args.eval_interval == 0: with print_time('mx.nd.waitall()'): mx.nd.waitall() with print_time('evaluate'): evaluate(args, model, vocab, i + num_batches * epoch) # Evaluate with print_time('mx.nd.waitall()'): mx.nd.waitall() with print_time('evaluate'): evaluate(args, model, vocab, num_batches * args.epochs, eval_analogy=not args.no_eval_analogy) # Save params with print_time('save parameters'): model.save_parameters(os.path.join(args.logdir, 'glove.params'))
def __call__(self): self.data.update_before_computing_gradients() g_st = time.time() self.compute_gradients() g_ed = time.time() self.data.update_before_computing_natural_gradients() if self.state['lr_adapt'] == 1: if self.step > self.state['lr_adapt_start']: if self.step % self.state['lr_adapt_change'] == 0: self.lr = self.lr * self.state['lr_adapt_decrease'] elif self.state['lr_adapt'] == 2: if self.step > self.state['lr_adapt_start']: self.lr = self.state['lr0'] /\ (1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta']) self.state['lr'] = float(self.lr) if self.step % self.state['trainFreq'] == 0: e_st = time.time() old_cost = self.compute_old_cost() new_cost, error = self.compute_new_cost(self.lr) rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr) new_cost, error = self.compute_new_cost_all(self.lr) if new_cost > self.state['btraincost'] * 6: raise Exception('Variance too large on training cost!') while (numpy.isnan(new_cost) or numpy.isinf(new_cost)): raise Exception('Learning rate too small !') self.old_cost = new_cost self.update_params(self.lr) e_ed = time.time() self._ocost = old_cost self._ncost = new_cost self._error = error self._eetime = e_ed self._estime = e_st msg = ('.. iter %4d cost %.3g, error %.3g step_size %.3g ' 'rho %.3g ' 'norm grad %.3g ' 'time [grad] %s,' '[updates param] %s,' 'whole time %s' ) print msg % ( self.step, new_cost, error, self.lr, rho, norm_grad, print_time(g_ed - g_st), print_time(e_ed - e_st), print_time(time.time() - self.step_timer) ) self.step_timer = time.time() else: old_cost = self._ocost new_cost = self._ncost error = self._error e_ed = self._eetime e_st = self._estime rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr) self.update_params(self.lr) self.step += 1 ret = { 'cost': float(new_cost), 'error': float(error), 'time_grads': float(g_ed - g_st), 'time_eval': float(e_ed - e_st), 'norm_grad':norm_grad, 'lr': self.lr, 'rho': numpy.float32(rho)} return ret
def main(stdfeat, num_epochs=300, param_file=None, model_file=None, testing_type='8k', model='jy', frame_level='', continue_train=False, delta=False, gaussian=True): # gc.disable() # Load the dataset print("Loading data...") test_only = ( model_file and os.path.isfile(model_file) ) # train_batches, val_batches, test_batches = load_dataset(training_type, testing_type, # fold, augment, delta, test_only, model_file) # print('train batches: {}'.format(len(train_batches))) # print('val batches: {}'.format(len(val_batches))) # print('test batches: {}'.format(len(test_batches))) # for bats, idx in my_iterator(train_batches, 100): # np.save('var/stdfeat/train_'+str(idx), bats) # for bats, idx in my_iterator(val_batches, 100): # np.save('var/stdfeat/val_'+str(idx), bats) # for bats, idx in my_iterator(test_batches, 100): # np.save('var/stdfeat/test_'+str(idx), bats) # return test_batches = get_test_batches(stdfeat) # build networks and functions train_fn, val_fn, network = init_process(model, gaussian, delta) if not test_only or continue_train: if continue_train and os.path.isfile(model_file): print('Continue training {}'.format(model_file)) val = np.load(model_file) lasagne.layers.set_all_param_values(network, val) print("Starting training...") lowest_err = 100000.0 highest_acc = 0.0 temp_err_filename = '.temp_err_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \ else '.temp_err_{}'.format(os.path.basename(model_file)) temp_acc_filename = '.temp_acc_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \ else '.temp_acc_{}'.format(os.path.basename(model_file)) n_cores = 5 qlts = [] for i in range(n_cores): # mgr = Manager() qlts.append((collections.deque(), threading.Lock(), i)) # pool = Pool(processes=n_cores) # train_queue, val_queue = mgr.list(), mgr.list() # train_queue, val_queue = Queue(2000), Queue(1000) # tr_fp_list = get_fp_list(stdfeat, num_epochs, True) # tr_fp_list = get_fp_list('train', stdfeat, num_epochs, True) # va_fp_list = get_fp_list('val', stdfeat, num_epochs, True) # load_data_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list), # name='tr_loadfile') # load_train_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list), # name='tr_loadfile') # load_val_proc = Process(target=th_loadfile, args=(val_queue, va_fp_list), # name='va_loadfile') # load_data_proc.start() # load_train_proc.start() # load_val_proc.start() tr_num = get_data_num('train', stdfeat) va_num = get_data_num('val', stdfeat) sys.stdout.flush() for epoch in range(num_epochs): print('epoch {}. {}'.format(epoch, utils.print_time())) start_time = time.time() # np.random.shuffle(train_batches) train_err, no_tr, val_err, val_acc, no_va = 0, 0, 0, 0, 0 out_list, tar_list = [], [] result_map = np.zeros((10,10), dtype=np.int32) if cache_data and epoch != 0: np.random.shuffle(all_data_queues['train']) for batch in all_data_queues['train']: err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn) if err is not None: train_err += err*len(pred) no_tr += len(pred) # print('Total number of training data: {}'.format(no_tr)) for batch in all_data_queues['val']: err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn) if err is not None: val_err += err*len(pred) val_acc += acc*len(pred) no_va += len(pred) for i, j in zip(targets, pred): result_map[i.argmax()][j] += 1 # print('Total number of validation data: {}'.format(no_va)) else: # pool.map_async(pool_loadfile, gen_pool_fp_list('train', stdfeat, qlts, True)) proc_list = gen_procs(qlts, generate_fp_list('train', stdfeat, True), 'train') for batch in new_iter_batch(qlts, tr_num): # for batch in iter_batch(train_queue, 'train', tr_num): err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn) if err is not None: train_err += err*len(pred) no_tr += len(pred) # print('Join all procs.') for proc in proc_list: proc.join() print('Total number of training data: {}'.format(no_tr)) # pool.map_async(pool_loadfile, gen_pool_fp_list('val', stdfeat, qlts, False)) proc_list = gen_procs(qlts, generate_fp_list('val', stdfeat, True), 'val') for batch in new_iter_batch(qlts, va_num): # for batch in iter_batch(train_queue, 'val', va_num): # for batch in iter_batch(val_queue, 'val', va_num): err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn) # print('out:\n {}'.format(out)) # print('targets:\n {}'.format(targets)) if err is not None: val_err += err*len(pred) val_acc += acc*len(pred) no_va += len(pred) # print((err, acc, pred_prob)) for i, j in zip(targets, pred): result_map[i.argmax()][j] += 1 # print('Join all procs.') for proc in proc_list: proc.join() print('Total number of validation data: {}'.format(no_va)) # Print the results for this epoch: print("Epoch {} of {} took {:.3f}s".format( epoch + 1, num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / no_tr)) print(" validation loss:\t\t{:.6f}".format(val_err / no_va)) print(" validation accuracy:\t\t{:.2f} %".format( val_acc / no_va * 100)) print("Result map: (x: prediction, y: target)") print(result_map) # Save the best model if (val_err / no_va) <= lowest_err: lowest_err = val_err / no_va final_err_param = lasagne.layers.get_all_param_values(network) np.save(temp_err_filename, final_err_param) if (val_acc / no_va) >= highest_acc: highest_acc = val_acc / no_va final_acc_param = lasagne.layers.get_all_param_values(network) np.save(temp_acc_filename, final_acc_param) if epoch%25 == 24: run_test(test_batches, val_fn, testing_type, frame_level, print_prob=True) sys.stdout.flush() # load_train_proc.join() # load_val_proc.join() # pool.close() # pool.join() if model_file: np.save('model/final/' + model_file, lasagne.layers.get_all_param_values(network)) # return to the best weights val = np.load(temp_acc_filename) lasagne.layers.set_all_param_values(network, val) if model_file: os.rename(temp_acc_filename, 'model/acc/' + model_file) os.rename(temp_err_filename, 'model/err/' + model_file) else: print('Load weights from file {}'.format(model_file)) val = np.load(model_file) lasagne.layers.set_all_param_values(network, val) run_test(test_batches, val_fn, testing_type, frame_level)
def __call__(self): self.data.update_before_computing_gradients() g_st = time.time() new_cost = self.compute_gradients() g_ed = time.time() self.data.update_before_computing_natural_gradients() r_st = time.time() rvals = self.compute_natural_gradients() r_ed = time.time() self.data.update_before_evaluation() e_st = time.time() old_cost = self.compute_old_cost() new_cost, error = self.compute_new_cost(self.lr) rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr, rvals[5]*rvals[6]) if self.step < 1: rho = .6 if self.state['adapt'] == 1: if rho < .25: self.damping.set_value(numpy.float32( self.damping.get_value() * self.state['damp_ratio'])) elif rho > .75 and self.damping.get_value() > self.state['mindamp']: self.damping.set_value(numpy.float32( self.damping.get_value() / self.state['damp_ratio'])) e_ed = time.time() if new_cost >= old_cost: print ('Variance too large on training cost!') self.damping.set_value(numpy.float32( self.damping.get_value() + 1.)) else: self.update_params(self.lr) print 'Minres: %s' % self.msgs[rvals[0]], \ '# iters %04d' % rvals[1], \ 'relative error residuals %.4g' % rvals[2], \ 'Anorm', rvals[3], 'Acond', rvals[4] msg = ('.. iter %4d ' 'cost %.4e ' 'old_cost %.4e ' 'step_size %.1e ' ' damping %.1e ' 'ord0_norm %.1e ' 'norm grad %.1e ' 'norm nat grad %.1e ' 'angle %.1e ' 'rho %.1e ' 'time [grad] %s,' '[riemann grad] %s,' '[updates param] %s,' 'whole time %s') print msg % ( self.step, new_cost, old_cost, self.lr, self.damping.get_value(), rvals[7], rvals[5], rvals[6], angle, rho, print_time(g_ed - g_st), print_time(r_ed - r_st), print_time(e_ed - e_st), print_time(time.time() - self.step_timer)) self.step_timer = time.time() self.old_cost = new_cost e_ed = time.time() self.step += 1 ret = { 'cost': float(new_cost), 'old_cost':float(old_cost), 'error': float(error), 'time_grads': float(g_ed - g_st), 'time_metric': float(r_ed - r_st), 'time_eval': float(e_ed - e_st), 'minres_flag': rvals[0], 'minres_iters': rvals[1], 'minres_relres': rvals[2], 'minres_Anorm': rvals[3], 'minres_Acond': rvals[4], 'norm_ord0': rvals[7], 'norm_grad':rvals[5], 'norm_nat': rvals[6], 'grad_angle' : float(angle), 'rho': float(rvals[-1]), 'lr': self.lr, #'r_g': float(r_g), #'icost' : float(tmp_cost), 'damping': self.damping.get_value(), 'rho': numpy.float32(rho) } return ret
def main(stdfeat, num_epochs=300, param_file=None, model_file=None, testing_type='8k', model='jy', frame_level='', continue_train=False, delta=False, gaussian=True): # gc.disable() # Load the dataset print("Loading data...") test_only = ( model_file and os.path.isfile(model_file) ) # train_batches, val_batches, test_batches = load_dataset(training_type, testing_type, # fold, augment, delta, test_only, model_file) # print('train batches: {}'.format(len(train_batches))) # print('val batches: {}'.format(len(val_batches))) # print('test batches: {}'.format(len(test_batches))) # for bats, idx in my_iterator(train_batches, 100): # np.save('var/stdfeat/train_'+str(idx), bats) # for bats, idx in my_iterator(val_batches, 100): # np.save('var/stdfeat/val_'+str(idx), bats) # for bats, idx in my_iterator(test_batches, 100): # np.save('var/stdfeat/test_'+str(idx), bats) # return mgr = Manager() shared_param = mgr.Namespace() shared_param.m_type = model shared_param.gaussian = gaussian shared_param.delta = delta model_lock = mgr.Lock() if not test_only or continue_train: if continue_train and os.path.isfile(model_file): print('Continue training {}'.format(model_file)) val = np.load(model_file) shared_param.model = val print("Starting training...") lowest_err = 100000.0 highest_acc = 0.0 temp_err_filename = '.temp_err_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \ else '.temp_err_{}'.format(os.path.basename(model_file)) temp_acc_filename = '.temp_acc_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \ else '.temp_acc_{}'.format(os.path.basename(model_file)) n_cores = 5 qlts = [] # global cache_data_dict # cache_data_dict = {'train':{}, 'val':{}} # global cache_lock # cache_lock = threading.Lock() # shared_param = mgr.dict({'m_type': model, 'gaussian':gaussian, 'delta':delta, # 'train_err':0, 'no_tr':0, 'val_acc':0, 'val_err':0, 'no_va':0}) for i in range(n_cores): # mgr = Manager() qlts.append((mgr.Queue(200), i)) # pool = Pool(processes=n_cores) # train_queue, val_queue = mgr.list(), mgr.list() # train_queue, val_queue = Queue(2000), Queue(1000) # tr_fp_list = get_fp_list(stdfeat, num_epochs, True) # tr_fp_list = get_fp_list('train', stdfeat, num_epochs, True) # va_fp_list = get_fp_list('val', stdfeat, num_epochs, True) # load_data_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list), # name='tr_loadfile') # load_train_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list), # name='tr_loadfile') # load_val_proc = Process(target=th_loadfile, args=(val_queue, va_fp_list), # name='va_loadfile') # load_data_proc.start() # load_train_proc.start() # load_val_proc.start() tr_num = get_data_num('train', stdfeat) va_num = get_data_num('val', stdfeat) sys.stdout.flush() for epoch in xrange(num_epochs): print('epoch {}. {}'.format(epoch+1, utils.print_time())) start_time = time.time() # np.random.shuffle(train_batches) # train_err = 0 # no_tr = 0 shared_param.finished_num = 0 shared_param.train_err = 0 shared_param.no_tr = 0 # pool.map_async(pool_loadfile, gen_pool_fp_list('train', stdfeat, qlts, True)) proc_list = gen_procs(n_cores, model_lock, generate_fp_list('train', stdfeat, True), shared_param, 'train') # for batch in new_iter_batch(qlts, tr_num, 'train'): # # for batch in iter_batch(train_queue, 'train', tr_num): # err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn) # if err is not None: # train_err += err*len(pred) # no_tr += len(pred) # print('Join all procs.') for proc in proc_list: proc.join() print('Total number of training data: {}'.format(shared_param.no_tr)) shared_param.val_err = 0 shared_param.val_acc = 0 out_list, tar_list = [], [] shared_param.result_map = np.zeros((10,10), dtype=np.int32) shared_param.no_va = 0 # pool.map_async(pool_loadfile, gen_pool_fp_list('val', stdfeat, qlts, False)) proc_list = gen_procs(n_cores, model_lock, generate_fp_list('val', stdfeat, True), shared_param, 'val') # for batch in new_iter_batch(qlts, va_num, 'val'): # # for batch in iter_batch(train_queue, 'val', va_num): # # for batch in iter_batch(val_queue, 'val', va_num): # err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn) # # print('out:\n {}'.format(out)) # # print('targets:\n {}'.format(targets)) # if err is not None: # val_err += err*len(pred) # val_acc += acc*len(pred) # no_va += len(pred) # # print((err, acc, pred_prob)) # for i, j in zip(targets, pred): # result_map[i.argmax()][j] += 1 # print('Join all procs.') for proc in proc_list: proc.join() print('Total number of validation data: {}'.format(shared_param.no_va)) # Print the results for this epoch: train_err, no_tr, result_map = shared_param.train_err, shared_param.no_tr, shared_param.result_map val_acc, val_err, no_va = shared_param.val_acc, shared_param.val_err, shared_param.no_va print("Epoch {} of {} took {:.3f}s".format( epoch + 1, num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / no_tr)) print(" validation loss:\t\t{:.6f}".format(val_err / no_va)) print(" validation accuracy:\t\t{:.2f} %".format( val_acc / no_va * 100)) print("Result map: (x: prediction, y: target)") print(result_map) # Save the best model if (val_err / no_va) <= lowest_err: lowest_err = val_err / no_va final_err_param = shared_param.model np.save(temp_err_filename, final_err_param) if (val_acc / no_va) >= highest_acc: highest_acc = val_acc / no_va final_acc_param = shared_param.model np.save(temp_acc_filename, final_acc_param) if epoch%25 == 24: gen_test_proc(shared_param, stdfeat, testing_type, frame_level, print_prob=True) sys.stdout.flush() # load_train_proc.join() # load_val_proc.join() # pool.close() # pool.join() if model_file: np.save('model/final/' + model_file, shared_param.model) # return to the best weights val = np.load(temp_acc_filename) shared_param.model = val if model_file: os.rename(temp_acc_filename, 'model/acc/' + model_file) os.rename(temp_err_filename, 'model/err/' + model_file) else: print('Load weights from file {}'.format(model_file)) val = np.load(model_file) shared_param.model = val gen_test_proc(shared_param, stdfeat, testing_type, frame_level)