def compute_perplexity(model, sess, name):
"""Compute perplexity of the output of the model.
Args:
model: model for compute perplexity.
sess: tensorflow session to use.
name: name of the batch.
Returns:
The perplexity of the eval outputs.
"""
total_loss = 0
total_predict_count = 0
start_time = time.time()
step = 0
while True:
try:
loss, predict_count, batch_size = model.eval(sess)
total_loss += loss * batch_size
total_predict_count += predict_count
step += 1
if step % 500 == 0:
ls = total_loss / total_predict_count
ppl = misc.safe_exp(ls)
print_out(" ## After %d steps, loss %.2f - ppl %.3f" %
(step, ls, ppl))
except tf.errors.OutOfRangeError:
break
perplexity = safe_exp(total_loss / total_predict_count)
print_time(" eval %s: perplexity %.2f" % (name, perplexity), start_time)
return perplexity
def load_embedding_from_path(args):
"""Load a TokenEmbedding."""
if 'bin' in args.embedding_path:
with utils.print_time('load fastText model.'):
model = \
nlp.model.train.FasttextEmbeddingModel.load_fasttext_format(
args.embedding_path)
embedding = nlp.embedding.TokenEmbedding(
unknown_token=None, unknown_lookup=model, allow_extend=True,
unknown_autoextend=True)
idx_to_token = sorted(model._token_to_idx, key=model._token_to_idx.get)
if not args.analogy_datasets:
# Prune tokens not used in evaluation datasets
eval_tokens_ = set(
evaluation.get_tokens_in_evaluation_datasets(args))
idx_to_token = [t for t in idx_to_token if t in eval_tokens_]
if args.max_vocab_size:
idx_to_token = idx_to_token[:args.max_vocab_size]
with utils.print_time('compute vectors from subwords '
'for {} words.'.format(len(idx_to_token))):
embedding[idx_to_token] = model[idx_to_token]
else:
embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path)
return embedding
def compute_perplexity(model, sess, name):
"""Compute perplexity of the output of the model.
Args:
model: model for compute perplexity.
sess: tensorflow session to use.
name: name of the batch.
Returns:
The perplexity of the eval outputs.
"""
total_loss = 0
total_predict_count = 0
start_time = time.time()
while True:
try:
output_tuple = model.eval(sess)
total_loss += output_tuple.eval_loss * output_tuple.batch_size
total_predict_count += output_tuple.predict_count
except tf.errors.OutOfRangeError:
break
perplexity = utils.safe_exp(total_loss / total_predict_count)
utils.print_time(" eval %s: perplexity %.2f" % (name, perplexity),
start_time)
return perplexity
def analyze(self):
trf = 'transforms.trf'
cfg = self.cfg
input_video = cfg.convey.curr_input_video
vidstab_dir = cfg.convey.curr_vidstab_dir
step = 'stepsize='+str(cfg.args.vs_stepsize)
mincontrast = float(cfg.args.vs_mincontrast)
detect = 'vidstabdetect=shakiness=10:accuracy=15:{0}:mincontrast={1}:result={2}:show=1'
filts = detect.format(step, mincontrast, trf)
show_detect = path.join(vidstab_dir, 'show.mkv')
#skip_first = "select='gte(n,0)',"
cmd = ['ffmpeg', '-i', input_video, '-c:v', 'libx264', '-crf', '18',
'-vf', filts,
'-an', '-y',
'-loglevel', 'error',
'-stats',
show_detect]
print('Analyze cam motions in video (libvidstab)')
if cfg.args.verbose:
print(' '.join(cmd))
s = dt.datetime.now()
run(cmd, cwd=vidstab_dir)
e = dt.datetime.now() - s
utils.print_time(e.total_seconds())
## saves global_motions.trf
## was needed before gradient descent impl in py from C
self.save_global_motions_trf_file(input_video, vidstab_dir)
def analyze2(self):
trf = 'transforms.trf'
cfg = self.cfg
if utils.args_rolling_shutter():
self.create_processed_vidstab_input(cfg.input_processed_video_path)
step = 'stepsize='+str(cfg.args.vs_stepsize)
mincontrast = float(cfg.args.vs_mincontrast)
detect = 'vidstabdetect=shakiness=10:accuracy=15:{0}:mincontrast={1}:result={2}:show=1'
filts = detect.format(step, mincontrast, trf)
input_video = cfg.convey.curr_input_video
vidstab_dir = cfg.convey.curr_vidstab_dir
show_detect = path.join(vidstab_dir, 'show.mkv')
cmd = ['ffmpeg', '-i', input_video, '-c:v', 'libx264', '-crf', '18',
'-vf', filts,
'-an', '-y',
'-loglevel', 'error',
'-stats',
show_detect]
s = dt.datetime.now()
print('Vidstab analyze video after rolling shutter correction')
if cfg.args.verbose:
print(' '.join(cmd))
print('FFMPEG output:')
run(cmd, cwd=vidstab_dir)
else:
run(cmd, cwd=vidstab_dir, stdout=DEVNULL)
e = dt.datetime.now() - s
utils.print_time(e.total_seconds())
def train(self, epochs_to_train=5):
meta = self.get_meta()
ut.print_time('train started: \n%s' % ut.to_file_name(meta))
ut.configure_folders(FLAGS, meta)
self.fetch_datasets(self._activation)
self.build_model()
self._register_training_start()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
self._saver = tf.train.Saver()
if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()):
self._saver.restore(sess, self.get_checkpoint_path())
ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31)
# MAIN LOOP
for current_epoch in xrange(epochs_to_train):
start = time.time()
feed = self._get_epoch_dataset()
for _, batch in enumerate(feed):
encoding, reconstruction, loss, _, _ = sess.run(
[self._encode, self._decode, self._reco_loss, self._train, self._step],
feed_dict={self._input: batch[0], self._reconstruction: batch[0]})
self._register_batch(loss)
self._register_epoch(current_epoch, epochs_to_train, time.time()-start, sess)
self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph)
meta = self._register_training()
return meta, self._stats['epoch_accuracy']
def load_embedding_from_path(args):
"""Load a TokenEmbedding."""
if 'bin' in args.embedding_path:
with utils.print_time('load fastText model.'):
model = \
nlp.model.train.FasttextEmbeddingModel.load_fasttext_format(
args.embedding_path)
# Add OOV words if the token_embedding can impute them
token_set = set()
token_set.update(
filter(lambda x: x in model,
evaluation.get_tokens_in_evaluation_datasets(args)))
# OOV words will be imputed and added to the
# token_embedding.idx_to_token etc.
with utils.print_time('compute vectors from subwords '
'for {} words.'.format(len(token_set))):
embedding = nlp.embedding.TokenEmbedding(unknown_token=None,
allow_extend=True)
idx_to_tokens = list(token_set)
embedding[idx_to_tokens] = model[idx_to_tokens]
else:
embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path)
return embedding
def frames(self):
cfg = self.cfg
pto_files = sorted(os.listdir(cfg.convey.curr_hugin_ptos_dir))
all_out_frames = cfg.convey.all_out_frames
tasks = []
for i, pto in enumerate(pto_files):
tasks.append(
datatypes.hugin_task(str(i + 1).zfill(6) + '.jpg', pto))
self.prjn_frames_total = len(tasks)
self.prjn_frames_cnt = 0
s = dt.datetime.now()
with Pool(int(cfg.args.num_cpus)) as p:
print(f'Create stabilized frames for output video.')
results = [
p.apply_async(hugin.frames_output,
args=(t, all_out_frames, cfg.frames_stabilized),
callback=self.prjn_worker_callback)
for t in tasks
]
for r in results:
r.get()
e = dt.datetime.now() - s
utils.print_time(e.total_seconds(), fps=True)
def load_embedding_from_path(args):
"""Load a TokenEmbedding."""
if args.embedding_path.endswith('.bin'):
with utils.print_time('load fastText model.'):
model = \
nlp.model.train.FasttextEmbeddingModel.load_fasttext_format(
args.embedding_path)
idx_to_token = sorted(model._token_to_idx, key=model._token_to_idx.get)
embedding = nlp.embedding.TokenEmbedding(
unknown_token=None, unknown_lookup=model, allow_extend=True)
# Analogy task is open-vocabulary, so must keep all known words.
# But if not evaluating analogy, no need to precompute now as all
# words for closed vocabulary task can be obtained via the unknown
# lookup
if not args.analogy_datasets:
idx_to_token = []
elif args.analogy_datasets and args.analogy_max_vocab_size:
idx_to_token = idx_to_token[:args.analogy_max_vocab_size]
embedding['<unk>'] = mx.nd.zeros(model.weight.shape[1])
if idx_to_token:
with utils.print_time('compute vectors for {} known '
'words.'.format(len(idx_to_token))):
embedding[idx_to_token] = model[idx_to_token]
else:
embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path)
return embedding
def zscore_signal(writer, original, save_params):
# Retrieve/generate zscore params
if os.path.isfile(save_params):
params = np.load(save_params)
normalizer = SignalNormalizer("zscore",
mu=params["mu"],
std=params["std"])
else:
mu = np.mean(original)
std = np.std(original)
normalizer = SignalNormalizer("zscore",
save_params=save_params,
mu=mu,
std=std)
print_time(
"mu={:.4f}, std={:.4f} obtained from {}".format(
normalizer.mu, normalizer.std, save_params),
start_time,
)
zscore = normalizer.normalize(original)
# Save zscored signal to disk.
for path in writer.paths:
path = path.replace(".npz",
".threshold{:.0e}.npz".format(args.threshold))
batch = dict(np.load(path))
batch["zscore"] = zscore[batch["start"]:batch["stop"]]
np.savez_compressed(path, **batch)
def main(args):
utils.print_time(start=True)
args.path.checkpoint, args.wandb.notes = utils.make_directories(args)
if args.wandb.log:
wandb.init(project=args.wandb.project,
name=args.wandb.notes,
config=args.config,
notes=args.wandb.notes,
allow_val_change=True)
utils.save_code(args)
print('=' * 100)
print('Arguments =')
for arg in vars(args):
print('\t' + arg + ':', getattr(args, arg))
print('=' * 100)
for n in range(args.train.num_runs):
args.seed = n + 1
args.experiment.memory_budget = int(args.experiment.memory_budget)
args.path.output = 'Run_{}_{}.txt'.format(n + 1, args.wandb.notes)
if args.wandb.log:
wandb.config.update(args, allow_val_change=True)
print(">" * 30, "Run #", n + 1)
run(args, n)
print("All Done! ")
print('[Elapsed time = {:.1f} min - {:0.1f} hours]'.format(
(time.time() - tstart) / (60), (time.time() - tstart) / (3600)))
utils.print_time(start=False)
def load_embedding_from_path(args):
"""Load a TokenEmbedding."""
if 'bin' in args.embedding_path:
with utils.print_time('load fastText model.'):
model = \
nlp.model.train.FasttextEmbeddingModel.load_fasttext_format(
args.embedding_path)
embedding = nlp.embedding.TokenEmbedding(
unknown_token=None, unknown_lookup=model, allow_extend=True,
unknown_autoextend=True)
idx_to_token = sorted(model._token_to_idx, key=model._token_to_idx.get)
if not args.analogy_datasets:
# Prune tokens not used in evaluation datasets
eval_tokens_ = set(
evaluation.get_tokens_in_evaluation_datasets(args))
idx_to_token = [t for t in idx_to_token if t in eval_tokens_]
if args.max_vocab_size:
idx_to_token = idx_to_token[:args.max_vocab_size]
with utils.print_time('compute vectors from subwords '
'for {} words.'.format(len(idx_to_token))):
embedding[idx_to_token] = model[idx_to_token]
else:
embedding = nlp.embedding.TokenEmbedding.from_file(args.embedding_path)
return embedding
def _register_training(self):
best_acc = np.min(self._stats['epoch_accuracy'])
meta = self.get_meta()
meta['acu'] = int(best_acc)
meta['e'] = self.get_past_epochs()
ut.print_time('Best Quality: %f for %s' % (best_acc, ut.to_file_name(meta)))
return meta
def main(args):
utils.make_directories(args)
utils.some_sanity_checks(args)
utils.save_code(args)
print('=' * 100)
print('Arguments =')
for arg in vars(args):
print('\t' + arg + ':', getattr(args, arg))
print('=' * 100)
accuracies, forgetting = [], []
for n in range(args.num_runs):
args.seed = n
args.output = '{}_{}_tasks_seed_{}.txt'.format(args.experiment, args.ntasks, args.seed)
print ("args.output: ", args.output)
print (" >>>> Run #", n)
acc, bwt = run(args, n)
accuracies.append(acc)
forgetting.append(bwt)
print('*' * 100)
print ("Average over {} runs: ".format(args.num_runs))
print ('AVG ACC: {:5.4f}% \pm {:5.4f}'.format(np.array(accuracies).mean(), np.array(accuracies).std()))
print ('AVG BWT: {:5.2f}% \pm {:5.4f}'.format(np.array(forgetting).mean(), np.array(forgetting).std()))
print ("All Done! ")
print('[Elapsed time = {:.1f} min]'.format((time.time()-tstart)/(60)))
utils.print_time()
def _decode_inference_indices(model, sess, output_infer,
output_infer_summary_prefix, inference_indices,
tgt_eos, subword_option):
"""Decoding only a specific set of sentences."""
utils.print_out(" decoding to output %s , num sents %d." %
(output_infer, len(inference_indices)))
start_time = time.time()
with codecs.getwriter("utf-8")(tf.gfile.GFile(output_infer,
mode="wb")) as trans_f:
trans_f.write("") # Write empty string to ensure file is created.
for decode_id in inference_indices:
nmt_outputs, infer_summary = model.decode(sess)
# get text translation
assert nmt_outputs.shape[0] == 1
translation = nmt_utils.get_translation(
nmt_outputs,
sent_id=0,
tgt_eos=tgt_eos,
subword_option=subword_option)
if infer_summary is not None: # Attention models
image_file = output_infer_summary_prefix + str(
decode_id) + ".png"
utils.print_out(" save attention image to %s*" % image_file)
image_summ = tf.Summary()
image_summ.ParseFromString(infer_summary)
with tf.gfile.GFile(image_file, mode="w") as img_f:
img_f.write(image_summ.value[0].image.encoded_image_string)
trans_f.write("%s\n" % translation)
utils.print_out(translation + b"\n")
utils.print_time(" done", start_time)
def get_train_data(args):
"""Helper function to get training data."""
with print_time('load training dataset'):
dataset = nlp.data.Text8(segment='train')
with print_time('count tokens'):
counter = nlp.data.count_tokens(itertools.chain.from_iterable(dataset))
vocab = nlp.Vocab(counter,
unknown_token=None,
padding_token=None,
bos_token=None,
eos_token=None,
min_freq=5)
idx_to_counts = mx.nd.array([counter[w] for w in vocab.idx_to_token])
negatives_weights = idx_to_counts**0.75
negatives_sampler = nlp.data.UnigramCandidateSampler(
weights=negatives_weights)
# Skip "unknown" tokens
with print_time('code dataset'):
coded_dataset = [[
vocab[token] for token in sentence if token in vocab
] for sentence in dataset]
with print_time('prune frequent words from sentences'):
frequent_tokens_subsampling_constant = 1e-3
f = idx_to_counts / mx.nd.sum(idx_to_counts)
idx_to_pdiscard = (
mx.nd.sqrt(frequent_tokens_subsampling_constant / f) +
frequent_tokens_subsampling_constant / f).asnumpy()
prune_sentences_ = functools.partial(prune_sentences,
idx_to_pdiscard=idx_to_pdiscard)
coded_dataset = list(map(prune_sentences_, coded_dataset))
with print_time('prepare subwords'):
subword_function = nlp.vocab.create_subword_function(
'NGramHashes', ngrams=args.ngrams, num_subwords=args.ngram_buckets)
# Precompute a idx to subwordidxs mapping to support fast lookup
idx_to_subwordidxs = list(subword_function(vocab.idx_to_token))
max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs)
# Padded max_subwordidxs_len + 1 so each row contains at least one -1
# element which can be found by np.argmax below.
idx_to_subwordidxs = np.stack(
np.pad(b.asnumpy(), (0, max_subwordidxs_len - len(b) + 1), \
constant_values=-1, mode='constant')
for b in idx_to_subwordidxs).astype(np.float32)
idx_to_subwordidxs = mx.nd.array(idx_to_subwordidxs)
logging.info(
'Using %s to obtain subwords. '
'The word with largest number of subwords '
'has %s subwords.', subword_function, max_subwordidxs_len)
return (coded_dataset, negatives_sampler, vocab, subword_function,
idx_to_subwordidxs)
def get_train_data(args):
"""Helper function to get training data."""
with print_time('load training dataset'):
dataset = nlp.data.Text8(segment='train')
with print_time('count tokens'):
counter = nlp.data.count_tokens(itertools.chain.from_iterable(dataset))
vocab = nlp.Vocab(counter,
unknown_token=None,
padding_token=None,
bos_token=None,
eos_token=None,
min_freq=5)
idx_to_counts = np.array([counter[w] for w in vocab.idx_to_token])
negatives_weights = idx_to_counts**0.75
negatives_sampler = nlp.data.UnigramCandidateSampler(
weights=mx.nd.array(negatives_weights))
# Skip "unknown" tokens
with print_time('code dataset'):
coded_dataset = [[
vocab[token] for token in sentence if token in vocab
] for sentence in dataset]
coded_dataset = [
sentence for sentence in coded_dataset if len(sentence)
]
with print_time('prune frequent words from sentences'):
f = idx_to_counts / np.sum(idx_to_counts)
idx_to_pdiscard = 1 - np.sqrt(args.frequent_token_subsampling / f)
prune_sentences_ = functools.partial(prune_sentences,
idx_to_pdiscard=idx_to_pdiscard)
coded_dataset = list(map(prune_sentences_, coded_dataset))
if args.ngram_buckets: # Fasttext model
with print_time('prepare subwords'):
subword_function = nlp.vocab.create_subword_function(
'NGramHashes',
ngrams=args.ngrams,
num_subwords=args.ngram_buckets)
# Store subword indices for all words in vocabulary
idx_to_subwordidxs = list(subword_function(vocab.idx_to_token))
get_subwords_masks = get_subwords_masks_factory(idx_to_subwordidxs)
max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs)
if max_subwordidxs_len > 500:
warnings.warn(
'The word with largest number of subwords '
'has {} subwords, suggesting there are '
'some noisy words in your vocabulary. '
'You should filter out very long words '
'to avoid memory issues.'.format(max_subwordidxs_len))
return (coded_dataset, negatives_sampler, vocab, subword_function,
get_subwords_masks)
else:
return coded_dataset, negatives_sampler, vocab
def compute_hugin_camera_rotations_processed(self):
cfg = self.cfg
if cfg.args.vidstab_prjn > -1:
vidstab_dir = cfg.prjn_dir2_vidstab_prjn
else:
vidstab_dir = cfg.prjn_dir2_vidstab_orig
if not utils.to_upd_camera_rotations_processed(vidstab_dir):
return
frames_dir = cfg.frames_input_processed
imgs = sorted(os.listdir(frames_dir))
self.canv_half_hfov = math.radians(self.rectilinear_pto.canv_half_hfov)
max_horizont_tan = math.tan(self.canv_half_hfov)
self.tan_pix = max_horizont_tan / (self.rectilinear_pto.canvas_w / 2)
##
# f_motions = open(path.join(vidstab_dir, 'global_motions.trf'))
# transforms_rel = utils.get_global_motions(f_motions)
s = dt.datetime.now()
transforms_rel = utils.parseTransformsTrf(
path.join(vidstab_dir, 'transforms.trf'))
e = dt.datetime.now() - s
utils.print_time(e.total_seconds(), fps=True)
##
transforms_abs = utils.convert_relative_transforms_to_absolute(
transforms_rel)
transforms_abs_filtered = utils.gauss_filter(cfg.fps, transforms_abs,
cfg.args.smoothing)
self.pto_txt = utils.create_pto_txt_one_image(cfg.pto.filepath)
tasks = []
for i, t in enumerate(zip(transforms_abs_filtered, transforms_rel)):
path_img = path.join(frames_dir, imgs[i])
tasks.append((t[0], path_img, t[1]))
utils.delete_files_in_dir(cfg.hugin_projects_processed)
self.prjn_frames_total = len(tasks)
self.prjn_frames_cnt = 0
s = dt.datetime.now()
with Pool(int(cfg.args.num_cpus)) as p:
print('Rolling shutter, create Hugin pto files with camera moves.')
results = [
p.apply_async(self.camera_moves_processed_worker,
args=(t, ),
callback=self.prjn_worker_callback)
for t in tasks
]
for r in results:
r.get()
e = dt.datetime.now() - s
utils.print_time(e.total_seconds(), fps=True)
def merge_signal(writer, seq_length):
original = np.empty((writer.counter, seq_length))
original[:] = np.NaN
for path in writer.paths:
batch = np.load(path)
print_time("[{}:{}]".format(batch["start"], batch["stop"]), start_time)
original[batch["start"] : batch["stop"]] = batch["original"]
return original
def train(self, epochs_to_train=5):
meta = self.get_meta()
ut.print_time('train started: \n%s' % ut.to_file_name(meta))
# return meta, np.random.randn(epochs_to_train)
ut.configure_folders(FLAGS, meta)
self._dataset, self._filters = self.fetch_datasets(self._activation)
self.build_model()
self._register_training_start()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
self._saver = tf.train.Saver()
if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()):
self._saver.restore(sess, self.get_checkpoint_path())
ut.print_info('Restored requested. Previous epoch: %d' %
self.get_past_epochs(),
color=31)
# MAIN LOOP
for current_epoch in xrange(epochs_to_train):
feed, permutation = self._get_epoch_dataset()
for _, batch in enumerate(feed):
filter = batch[1][0]
assert batch[1][0, 0] == batch[1][-1, 0]
encoding, = sess.run([self._encode],
feed_dict={self._input: batch[0]
}) # 1.1 encode forward
clamped_enc, vae_grad = _clamp(encoding,
filter) # 1.2 # clamp
sess.run(self._assign_clamped,
feed_dict={self._clamped: clamped_enc})
reconstruction, loss, clamped_gradient, _ = sess.run( # 2.1 decode forward+backward
[
self._decode, self._decoder_loss,
self._clamped_grad, self._train_decoder
],
feed_dict={
self._clamped: clamped_enc,
self._reconstruction: batch[0]
})
declamped_grad = _declamp_grad(
vae_grad, clamped_gradient,
filter) # 2.2 prepare gradient
_, step = sess.run( # 3.0 encode backward path
[self._train_encoder, self._step],
feed_dict={self._input: batch[0], self._encoding: encoding-declamped_grad}) # Profit
self._register_batch(batch, encoding, reconstruction, loss)
self._register_epoch(current_epoch, epochs_to_train,
permutation, sess)
self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph)
meta = self._register_training()
return meta, self._stats['epoch_accuracy']
def _register_training(self):
best_acc = np.min(self._stats['epoch_accuracy'])
meta = self.get_meta()
meta['acu'] = int(best_acc)
meta['e'] = self.get_past_epochs()
ut.print_time('Best Quality: %f for %s' %
(best_acc, ut.to_file_name(meta)))
self.summary_writer.close()
return meta
def __call__(self):
batch = self.data_iter.get_batch()
g_st = time.time()
self.grad_fn(*batch)
g_ed = time.time()
if self.state['lr_adapt'] == 1:
if self.step > self.state['lr_adapt_start']:
self.lr = self.state['lr0'] /\
(1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta'])
self.state['lr'] = float(self.lr)
e_st = time.time()
old_cost = self.compute_old_cost(*batch)
new_cost, error = self.compute_new_cost(self.lr, *batch)
rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr)
if new_cost > old_cost:
print ('Error increasing !? ')
self.lr = self.lr / 2.
while (numpy.isnan(new_cost) or
numpy.isinf(new_cost)):
raise Exception('Got Inf/NaN !')
self.old_cost = new_cost
self.update_params(self.lr)
e_ed = time.time()
msg = ('.. iter %4d cost %.3g (before update %.3g), error %.3g step_size %.3g '
'rho %.3g '
'norm grad %.3g '
'time [grad] %s,'
'[updates param] %s,'
'whole time %s'
)
print msg % (
self.step,
new_cost,
old_cost,
error,
self.lr,
rho,
norm_grad,
print_time(g_ed - g_st),
print_time(e_ed - e_st),
print_time(time.time() - self.step_timer) )
self.step_timer = time.time()
self.step += 1
ret = {
'cost': float(new_cost),
'error': float(error),
'time_grads': float(g_ed - g_st),
'time_eval': float(e_ed - e_st),
'norm_grad':norm_grad,
'lr': self.lr,
'rho': numpy.float32(rho)}
return ret
def merge_signal(writer, seq_length):
original = np.empty((writer.counter, seq_length, NUM_FEATURES))
original[:] = np.NaN
for path in writer.paths:
path = path.replace(".npz",
".threshold{:.0e}.npz".format(args.threshold))
batch = np.load(path)
print_time("[{}:{}]".format(batch["start"], batch["stop"]), start_time)
original[batch["start"]:batch["stop"]] = batch["original"]
return original
def main():
cmd_args, unknown_args = parser.parse_known_args()
cfg = config.cfg = config.Configuration(cmd_args)
inframes = cfg.inframes = InFrames(cfg)
vs = cfg.vidstab = Vidstab(cfg)
convey = cfg.convey = Conveyor(cfg)
out_frms = cfg.out_frms = out_frames.OutFrames(cfg)
## Cached previous command arguments
if not path.exists(cfg.cmd_args):
cmd_args.force_upd = True
open(cfg.cmd_args, 'w').write('\n'.join(sys.argv[1:]))
args_list = open(cfg.cmd_args, 'r').read().splitlines()
prev_parser = ArgumentParser(description='Previous arguments')
args.init_cmd_args(prev_parser)
cfg.prev_args, unknown_args_prev = prev_parser.parse_known_args(
args=args_list)
open(cfg.cmd_args, 'w').write('\n'.join(sys.argv[1:])) # update prev args
add_step = convey.add_step
step_arg = int(cmd_args.step)
print('Videofile: ', cfg.args.videofile)
print('Workdir: ', cfg.args.workdir)
print('PTO(Hugin): ', cfg.args.pto_name)
add_step('STEP_1: input frames and split audio.',
convey.to_upd_input_frames, inframes.store_input_frames)
add_step('STEP_2: analyze cam motions in input video.',
convey.to_upd_analyze, vs.analyze)
add_step('STEP_3: camera rotations in Hugin.',
convey.to_upd_camera_rotations,
out_frms.compute_hugin_camera_rotations)
add_step('STEP_4: analyze cam moves with corrected Rolling Shutter.',
convey.to_upd_rs_analyze, convey.rs_analyze)
add_step('STEP_5: create stabilized frames, Hugin.',
convey.to_upd_out_frames, out_frms.frames)
add_step('STEP_6: create video from stabilized frames.',
convey.to_upd_out_video, out_frms.video)
s = dt.datetime.now()
convey.execute(step_arg)
e = dt.datetime.now() - s
print()
print()
utils.print_time(e.total_seconds(), prefix='Total time')
def apply_gaussian(images, sigma):
if sigma == 0:
return images
# ut.print_time('start')
res = images.copy()
for i, image in enumerate(res):
for channel in range(image.shape[-1]):
image[:, :, channel] = filters.gaussian_filter(image[:, :, channel], sigma)
ut.print_time('finish s:%.1f' % sigma, images[2,10,10,0], res[2,10,10,0])
return res
async def scf_handle(reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
bridge = Conn("Bridge", reader, writer)
uid = await bridge.read(4)
uid = uid.decode("ascii")
client = uid_socket[uid]
bridge.target = client.target
bridge_addr, _ = bridge.writer.get_extra_info("peername")
print_time(f"Tencent IP:{bridge_addr} <=> {client.target} established")
await socks5_forward(client, bridge)
def visualize_encodings(encodings, file_name=None,
grid=None, skip_every=999, fast=False, fig=None, interactive=False):
encodings = manual_pca(encodings)
if encodings.shape[1] <= 3:
return print_data_only(encodings, file_name, fig=fig, interactive=interactive)
encodings = encodings[0:720]
hessian_euc = dist.squareform(dist.pdist(encodings[0:720], 'euclidean'))
hessian_cos = dist.squareform(dist.pdist(encodings[0:720], 'cosine'))
grid = (3, 4) if grid is None else grid
project_ops = []
n = 2
project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa')))
project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified')))
project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
project_ops.append(("TSNE 30/2000 N:%d" % n, TSNE(perplexity=30, n_components=n, init='pca', n_iter=2000)))
n = 3
project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa')))
project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified')))
project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
project_ops.append(('MDS cosine N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
plot_places = []
for i in range(12):
u, v = int(i / (skip_every - 1)), i % (skip_every - 1)
j = v + u * skip_every + 1
plot_places.append(j)
fig = get_figure(fig)
fig.set_size_inches(fig.get_size_inches()[0] * grid[0] / 1.,
fig.get_size_inches()[1] * grid[1] / 2.0)
for i, (name, manifold) in enumerate(project_ops):
is3d = 'N:3' in name
try:
if is3d:
subplot = plt.subplot(grid[0], grid[1], plot_places[i], projection='3d')
else:
subplot = plt.subplot(grid[0], grid[1], plot_places[i])
data_source = encodings if not _needs_hessian(manifold) else \
(hessian_cos if 'cosine' in name else hessian_euc)
projections = manifold.fit_transform(data_source)
scatter(subplot, projections, is3d, _build_radial_colors(len(data_source)))
subplot.set_title(name)
except:
print(name, "Unexpected error: ", sys.exc_info()[0], sys.exc_info()[1] if len(sys.exc_info()) > 1 else '')
visualize_data_same(encodings, grid=grid, places=plot_places[-4:])
if not interactive:
save_fig(file_name, fig)
ut.print_time('visualization finished')
def print_epoch_info(self, accuracy, current_epoch, epochs, elapsed):
epochs_past = self.get_past_epochs() - current_epoch
accuracy_info = '' if accuracy is None else '| accuracy %d' % int(
accuracy)
epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past -
1)
epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs,
epoch_past_info)
time_info = '%2dms/bt' % (elapsed / FLAGS.epoch_size * 1000)
info_string = ' '.join([epoch_count, accuracy_info, time_info])
ut.print_time(info_string, same_line=True)
def print_epoch_info(self, accuracy, current_epoch, epochs, elapsed):
epochs_past = self.get_past_epochs() - current_epoch
accuracy_info = '' if accuracy is None else '| accuracy %d' % int(accuracy)
epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past - 1)
epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs, epoch_past_info)
time_info = '%2dms/bt' % (elapsed / FLAGS.epoch_size*1000)
info_string = ' '.join([
epoch_count,
accuracy_info,
time_info])
ut.print_time(info_string, same_line=True)
def save_predictions_from_model(dicModel,
run_name='all_models',
output_folder=PATH_OUTPUT):
# create the filename
filename = '{}pred_{}_{}_{}.csv'.\
format(output_folder,
run_name,
int(dicModel['score_val'] * 1000),
dicModel['model'].__class__.__name__)
#save the predictions
pd.DataFrame(dict(preds=dicModel['pred_val'])) \
.to_csv(filename, index=False)
print_time('Saved {}'.format(filename))
def __call__(self):
batch = self.data_iter.get_batch()
g_st = time.time()
self.grad_fn(*batch)
g_ed = time.time()
if self.state['lr_adapt'] == 1:
if self.step > self.state['lr_adapt_start']:
self.lr = self.state['lr0'] /\
(1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta'])
self.state['lr'] = float(self.lr)
e_st = time.time()
old_cost = self.compute_old_cost(*batch)
new_cost, error = self.compute_new_cost(self.lr, *batch)
rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr)
if new_cost > old_cost:
print('Error increasing !? ')
self.lr = self.lr / 2.
while (numpy.isnan(new_cost) or numpy.isinf(new_cost)):
raise Exception('Got Inf/NaN !')
self.old_cost = new_cost
self.update_params(self.lr)
e_ed = time.time()
msg = (
'.. iter %4d cost %.3g (before update %.3g), error %.3g step_size %.3g '
'rho %.3g '
'norm grad %.3g '
'time [grad] %s,'
'[updates param] %s,'
'whole time %s')
print msg % (self.step, new_cost, old_cost, error, self.lr,
rho, norm_grad, print_time(g_ed - g_st),
print_time(e_ed - e_st),
print_time(time.time() - self.step_timer))
self.step_timer = time.time()
self.step += 1
ret = {
'cost': float(new_cost),
'error': float(error),
'time_grads': float(g_ed - g_st),
'time_eval': float(e_ed - e_st),
'norm_grad': norm_grad,
'lr': self.lr,
'rho': numpy.float32(rho)
}
return ret
def create_processed_vidstab_input(self, output):
print("Create corrected 'rolling shutter' video for vidstab")
crf = '16'
ivid = path.join(self.cfg.frames_input_processed, '%06d.jpg')
cmd = ['ffmpeg', '-framerate', self.cfg.fps, '-i', ivid,
'-c:v', 'libx264', '-crf', crf,
#'-vf', cropf,
'-loglevel', 'error', '-stats', '-an', '-y', output]
s = dt.datetime.now()
run(cmd)
e = dt.datetime.now() - s
utils.print_time(e.total_seconds())
def login(user):
"""Log the user into a deployment of Nexus."""
config = config_utils.get_cli_config()
c = config_utils.get_selected_deployment_config(config)
if c is None:
utils.error("You must select a deployment using `deployment --select` prior to running this command.")
active_deployment_cfg = c[1]
if user is None or user == '':
user = input("Username:"******"Login successful"))
if 'expires_in' in token:
access_token = jwt.decode(token['access_token'], verify=False)
expiry_utc = datetime.utcfromtimestamp(access_token['exp'])
print("Token is valid for %s, expiry at %s" % (utils.print_time(token['expires_in']),
utils.datetime_from_utc_to_local(expiry_utc)))
active_deployment_cfg['token'] = token
active_deployment_cfg['userinfo'] = userinfo
config_utils.save_cli_config(config)
def validate(self):
n_elems = 0
cost = 0
for batch in self.valid_data.__iter__():
n_elems += 1
cost += 100*self.model.validate(*batch)
cost /= numpy.float32(n_elems)
print ('** validation cost %6.3f computed in %s'
', best cost is %6.3f, test %6.3f, whole time %6.3f min') % (
cost,
print_time(time.time() - self.batch_start_time),
self.state['bvalidcost'],
self.state['testcost'],
(time.time() - self.start_time)/60. )
self.batch_start_time = time.time()
pos = self.step // self.state['validFreq']
self.timings['valid'][pos] = float(cost)
self.timings['test'][pos] = float(self.state['testcost'])
self.state['validcost'] = float(cost)
self.state['validtime'] = float(time.time() - self.start_time)/60.
if self.state['bvalidcost'] > cost:
self.state['bvalidcost'] = float(cost)
self.state['btraincost'] = float(self.state['traincost'])
self.test()
print_mem('validate')
def print(self) -> str:
s = max(self._bet_ref['finish_time'] - utils.now(), 0)
time_remaining_str = utils.print_time(self._lang.get(), s)
lines = [tr(self._lang.get(), 'BET.TIME', time=time_remaining_str)]
if self._info_changed:
self._stored_info = []
# Sort bet data
total_bet = self.get_bet_sum() + self._bot.get_bet()
bets = list(self._bet_ref['bets'].values())
bets.append((self._bot.icon, self._bot.get_bet()))
bets.sort(key=lambda x: x[1], reverse=True)
# Jackpot
a: str = tr(self._lang.get(),
'BET.JACKPOT',
EMOJI_SPARKLE=Emoji.SPARKLE,
money=utils.print_money(self._lang.get(), total_bet))
b: str = tr(self._lang.get(),
'BET.MAX_BET',
money=utils.print_money(self._lang.get(), self._limit))
self._stored_info.append(f"{a}\n{b}")
# Player+bot bets
for single_bet in bets:
money_str = utils.print_money(self._lang.get(), single_bet[1])
pct = single_bet[1] / total_bet
self._stored_info.append(
f"{single_bet[0]}: {money_str} ({pct:.0%})")
self._info_changed = False
return '\n'.join(lines + self._stored_info)
def iter_batch(batch_queue, data_type, num):
for i in range(num):
while True:
sleep_time = 0
# print('queue size: {}'.format(len(batch_queue)))
# print('queue size: {}'.format(batch_queue.qsize()))
while len(batch_queue) == 0:
# while batch_queue.empty():
# tt = 10 if batch_queue.qsize()==0 else 1
tt = 10
time.sleep(tt)
sleep_time += tt
# print('sleep {} seconds.'.format(sleep_time))
# if sleep_time > 0:
# print('sleep {} seconds.'.format(sleep_time))
bt = batch_queue.pop(0)
# bt = batch_queue.get(block=False)
if type(bt) != type('end'):
# print('get one batch!')
yield bt
else:
print('iter end one fp! {}'.format(utils.print_time()))
name = multiprocessing.current_process().name
print('{} current queue size: {}'.format(name, len(batch_queue)))
break
def _print_epoch_info(self, accuracy, current_epoch, epochs, elapsed):
epochs_past = self.get_past_epochs() - current_epoch
accuracy_info = '' if accuracy is None else '| accuracy %d' % int(
accuracy)
epoch_past_info = '' if epochs_past is None else '+%d' % (epochs_past -
1)
epoch_count = 'Epochs %2d/%d%s' % (current_epoch + 1, epochs,
epoch_past_info)
time_info = '%2dms/bt' % (elapsed / self.epoch_size * 1000)
examples = int(np.floor(len(self.train_set) / FLAGS.batch_size))
loss_info = 't.loss:%d' % (self.epoch_stats.total_loss * 100 /
(examples * np.prod(self.batch_shape[1:])))
info_string = ' '.join(
[epoch_count, accuracy_info, time_info, loss_info])
ut.print_time(info_string, same_line=True)
def grid_search(train_data, test_data, y_train, y_test, model,
seed=0, scorer=None, verbose=False, n_jobs=1,
val_size=.1):
'''
Grid Search the model. As the parameters are usually the same
in a grid search, they are stored here already, easing the use.
params:
train_data: train data. Will be split into train and validation
test_data: final test data
y_train: labels for train data
y_test: labels for the test data
model: model to be grid searched
'''
results = None
# split the train and validation data
X_train, X_val, y_train, y_val = cv.train_test_split(train_data,
y_train,
test_size=val_size,
random_state=seed)
# call the function according to model name
model_name = model.__class__.__name__
if model_name in ('GradientBoostingClassifier'):
param_grid = {'learning_rate': [0.1, 0.3],
'n_estimators': [200],
'max_depth': [3, 9, 18],
'subsample': [1],
'min_samples_leaf': [1, 7, 14, 20],
'max_features': [.4, .9, None],
}
# grid search Gradient Boosting
print_time('Starting grid search')
gbr_cv = GradientBoostingClassifier()
gs_cv = GridSearchCV(gbr_cv, param_grid,
verbose=verbose, n_jobs=n_jobs,
scoring=make_scorer(scorer)).\
fit(X_train, np.ravel(y_train))
gbr = gs_cv.best_estimator_
# validation dataset
pred_val = gbr.predict(X_val)
score_val = scorer(y_val, pred_val)
print_time('Validation prediction score: {}'.format(score_val))
# check the performance against the test data
pred_test = gbr.predict(test_data)
score_test = scorer(y_test, pred_test)
print_time('Test prediction score: {}'.format(score_test))
results = {'search': gs_cv, 'pred_test': pred_test,
'score_test': score_test, 'pred_val': pred_val,
'score_val': score_val}
print_time('Finished grid search')
return results
def text8(min_freq=5, max_vocab_size=None):
"""Text8 dataset helper.
Parameters
----------
min_freq : int, default 5
Minimum token frequency for a token to be included in the vocabulary
and returned DataStream.
max_vocab_size : int, optional
Specifies a maximum size for the vocabulary.
Returns
-------
gluonnlp.data.DataStream
Each sample is a valid input to
gluonnlp.data.EmbeddingCenterContextBatchify.
gluonnlp.Vocab
Vocabulary of all tokens in Text8 that occur at least min_freq times of
maximum size max_vocab_size.
idx_to_counts : list of int
Mapping from token indices to their occurrence-counts in the Text8
dataset.
"""
with print_time('read data'):
data = nlp.data.Text8(segment='train')
counter = nlp.data.count_tokens(itertools.chain.from_iterable(data))
vocab = nlp.Vocab(counter,
unknown_token=None,
padding_token=None,
bos_token=None,
eos_token=None,
min_freq=min_freq,
max_size=max_vocab_size)
idx_to_counts = [counter[w] for w in vocab.idx_to_token]
def code(sentence):
return [vocab[token] for token in sentence if token in vocab]
with print_time('code data'):
data = data.transform(code, lazy=False)
data = nlp.data.SimpleDataStream([data])
return data, vocab, idx_to_counts
def train(self, epochs_to_train=5):
meta = self.get_meta()
ut.print_time('train started: \n%s' % ut.to_file_name(meta))
# return meta, np.random.randn(epochs_to_train)
ut.configure_folders(FLAGS, meta)
self._dataset, self._filters = self.fetch_datasets(self._activation)
self.build_model()
self._register_training_start()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
self._saver = tf.train.Saver()
if FLAGS.load_state and os.path.exists(self.get_checkpoint_path()):
self._saver.restore(sess, self.get_checkpoint_path())
ut.print_info('Restored requested. Previous epoch: %d' % self.get_past_epochs(), color=31)
# MAIN LOOP
for current_epoch in xrange(epochs_to_train):
feed, permutation = self._get_epoch_dataset()
for _, batch in enumerate(feed):
filter = batch[1][0]
assert batch[1][0,0] == batch[1][-1,0]
encoding, = sess.run([self._encode], feed_dict={self._input: batch[0]}) # 1.1 encode forward
clamped_enc, vae_grad = _clamp(encoding, filter) # 1.2 # clamp
sess.run(self._assign_clamped, feed_dict={self._clamped:clamped_enc})
reconstruction, loss, clamped_gradient, _ = sess.run( # 2.1 decode forward+backward
[self._decode, self._decoder_loss, self._clamped_grad, self._train_decoder],
feed_dict={self._clamped: clamped_enc, self._reconstruction: batch[0]})
declamped_grad = _declamp_grad(vae_grad, clamped_gradient, filter) # 2.2 prepare gradient
_, step = sess.run( # 3.0 encode backward path
[self._train_encoder, self._step],
feed_dict={self._input: batch[0], self._encoding: encoding-declamped_grad}) # Profit
self._register_batch(batch, encoding, reconstruction, loss)
self._register_epoch(current_epoch, epochs_to_train, permutation, sess)
self._writer = tf.train.SummaryWriter(FLAGS.logdir, sess.graph)
meta = self._register_training()
return meta, self._stats['epoch_accuracy']
def preprocess_dataset(data, min_freq=5, max_vocab_size=None):
"""Dataset preprocessing helper.
Parameters
----------
data : mx.data.Dataset
Input Dataset. For example gluonnlp.data.Text8 or gluonnlp.data.Fil9
min_freq : int, default 5
Minimum token frequency for a token to be included in the vocabulary
and returned DataStream.
max_vocab_size : int, optional
Specifies a maximum size for the vocabulary.
Returns
-------
gluonnlp.data.DataStream
Each sample is a valid input to
gluonnlp.data.EmbeddingCenterContextBatchify.
gluonnlp.Vocab
Vocabulary of all tokens in Text8 that occur at least min_freq times of
maximum size max_vocab_size.
idx_to_counts : list of int
Mapping from token indices to their occurrence-counts in the Text8
dataset.
"""
with print_time('count and construct vocabulary'):
counter = nlp.data.count_tokens(itertools.chain.from_iterable(data))
vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None,
bos_token=None, eos_token=None, min_freq=min_freq,
max_size=max_vocab_size)
idx_to_counts = [counter[w] for w in vocab.idx_to_token]
def code(sentence):
return [vocab[token] for token in sentence if token in vocab]
with print_time('code data'):
data = data.transform(code, lazy=False)
data = nlp.data.SimpleDataStream([data])
return data, vocab, idx_to_counts
def train_regression(data, target, models=None,
scorer=None, verbose=False, seed=None):
'''
Helper to split the models always the same
:return: list of tuples (model id, dict of model)
:params:
data: data to be trained, no target values
target: model target val
models: models to be trained, inside a list []
scorer: the scorer to order the best results
verbose: print information
'''
seed = seed if seed else get_seed()
# get the models to test
if not models:
models = [ExtraTreesClassifier(random_state=seed),
GradientBoostingClassifier(random_state=seed),
RandomForestClassifier(random_state=seed),
LogisticRegressionCV(),
RidgeClassifierCV(),
LinearSVC(random_state=seed),
SVC(random_state=seed),
SGDClassifier(random_state=seed),
GaussianNB()]
# choose accuracy if no scorer was passed
if not scorer:
scorer = accuracy_score
# split in train and validation set
X_train, X_val, y_train, y_val = split_data(data, target, seed)
print_time('Created train and validation')
print_time('Size train: {} test:{}'.
format(X_train.shape, X_val.shape))
return train_all_models(X_train, X_val, y_train, y_val,
models, scorer, verbose)
def validate(self):
cost = self.model.validate()
print ('** validation cost %6.3f computed in %s'
', best cost is %6.3f, test %6.3f, whole time %6.3f min') % (
cost,
print_time(time.time() - self.batch_start_time),
self.state['bvalidcost'],
self.state['testcost'],
(time.time() - self.start_time)/60. )
self.batch_start_time = time.time()
pos = self.step // self.state['validFreq']
self.timings['valid'][pos] = float(cost)
self.timings['test'][pos] = float(self.state['testcost'])
self.state['validcost'] = float(cost)
self.state['validtime'] = float(time.time() - self.start_time)/60.
if self.state['bvalidcost'] > cost:
self.state['bvalidcost'] = float(cost)
self.state['btraincost'] = float(self.state['traincost'])
self.test()
print_mem('validate')
def th_loadfile(batch_queue, fp_list):
name = multiprocessing.current_process().name
for fp in fp_list:
while len(batch_queue) > 100:
# while batch_queue.full():
time.sleep(5)
# print('{} sleep for 60 seconds.'.format(name))
bts = np.load(fp)
# print('{} originally has {} batches.'.format(fp, len(bts)))
np.random.shuffle(bts)
# print('{} has {} batches to put.'.format(fp, len(bts)))
for bt in bts:
batch_queue.append(bt)
# batch_queue.put(bt, block=True)
batch_queue.append('end')
# batch_queue.put('end', block=True)
print('{} finished {}. {}'.format(name, fp, utils.print_time()))
print('{} current queue size: {}'.format(name, len(batch_queue)))
print('{} ends.'.format(name))
def main(self):
print_mem('start')
self.state['gotNaN'] = 0
self.start_time = time.time()
self.batch_start_time = time.time()
self.step = 0
self.save_iter = 0
self.save()
if self.channel is not None:
self.channel.save()
self.save_time = time.time()
last_cost = 1.
start_time = time.time()
self.start_time = start_time
while self.step < self.state['loopIters'] and \
last_cost > .1*self.state['minerr'] and \
(time.time() - start_time)/60. < self.state['timeStop']:
if (time.time() - self.save_time)/60. > self.state['saveFreq']:
self.save()
if self.channel is not None:
self.channel.save()
self.save_time = time.time()
st = time.time()
try:
rvals = self.algo()
self.state['traincost'] = float(rvals['cost'])
self.state['step'] = self.step
last_cost = rvals['cost']
for name in rvals.keys():
pos = self.step // self.state['trainFreq']
self.timings[name][pos] = rvals[name]
if numpy.isinf(rvals['cost']) or numpy.isnan(rvals['cost']):
self.state['gotNaN'] = 1
self.save()
if self.channel:
self.channel.save()
print 'Got NaN while training'
last_cost = 0
if self.step % self.state['validFreq'] == 0:
self.validate()
self.step += 1
except:
self.state['wholetime'] = float(time.time() - start_time)
self.save()
if self.channel:
self.channel.save()
last_cost = 0
print 'Error in running natgrad (lr issue)'
print 'BEST SCORE'
print 'Validation', self.state['validcost']
print 'Validation time', print_time(self.state['validtime'])
print 'Train cost', self.state['traincost']
print 'Best Train', self.state['btraincost']
print 'Best Valid', self.state['bvalidcost']
print 'TEST', self.state['testcost']
print 'Took', (time.time() - start_time)/60.,'min'
raise
self.state['wholetime'] = float(time.time() - start_time)
self.validate()
self.save()
if self.channel:
self.channel.save()
print 'BEST SCORE'
print 'Validation', self.state['validcost']
print 'Validation time', print_time(self.state['validtime'])
print 'Train cost', self.state['traincost']
print 'Best Train', self.state['btraincost']
print 'Best Valid', self.state['bvalidcost']
print 'TEST', self.state['testcost']
print 'Took', (time.time() - start_time)/60.,'min'
def visualize_encodings(encodings, file_name=None,
grid=None, skip_every=999, fast=False, fig=None, interactive=False):
# encodings = encodings[0:360] if len(encodings) < 1500 else encodings[0:720]
encodings = manual_pca(encodings)
if encodings.shape[1] <= 3:
return print_data_only(encodings, file_name, fig=fig, interactive=interactive)
encodings = encodings[0:720]
hessian_euc = dist.squareform(dist.pdist(encodings[0:720], 'euclidean'))
hessian_cos = dist.squareform(dist.pdist(encodings[0:720], 'cosine'))
grid = (3, 4) if grid is None else grid
project_ops = []
n = 2
project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa')))
project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified')))
project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
project_ops.append(("TSNE 30/2000 N:%d" % n, TSNE(perplexity=30, n_components=n, init='pca',n_iter=2000)))
n = 3
project_ops.append(("LLE ltsa N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='ltsa')))
project_ops.append(("LLE modified N:%d" % n, mn.LocallyLinearEmbedding(10, n, method='modified')))
project_ops.append(('MDS euclidean N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
project_ops.append(('MDS cosine N:%d' % n, mn.MDS(n, max_iter=300, n_init=1, dissimilarity='precomputed')))
# print(
# np.min(hessian_euc),
# np.min(hessian_cos),
# hessian_euc.size - np.count_nonzero(hessian_euc))
plot_places = []
for i in range(12):
u, v = int(i/(skip_every-1)), i % (skip_every - 1)
j = v + u * skip_every + 1
plot_places.append(j)
fig = fig if fig is not None else plt.figure()
fig.set_size_inches(fig.get_size_inches()[0] * grid[0] /1.,
fig.get_size_inches()[1] * grid[1]/2.0)
for i, (name, manifold) in enumerate(project_ops):
is3d = 'N:3' in name
# if (fast or 'grid' in FLAGS.suffix) and 'MDS' in name:
# continue
try:
if is3d: subplot = plt.subplot(grid[0], grid[1], plot_places[i], projection='3d')
else: subplot = plt.subplot(grid[0], grid[1], plot_places[i])
data_source = encodings if not _needs_hessian(manifold) else \
(hessian_cos if 'cosine' in name else hessian_euc)
projections = manifold.fit_transform(data_source)
scatter(subplot, projections, is3d, build_radial_colors(len(data_source)))
subplot.set_title(name)
except:
print(name, "Unexpected error: ", sys.exc_info()[0], sys.exc_info()[1] if len(sys.exc_info()) > 1 else '')
visualize_data_same(encodings, grid=grid, places=plot_places[-4:])
# visualize_data_same(encodings, grid=grid, places=np.arange(13, 17), dims_as_colors=True)
# fig.tight_layout()
if not interactive:
save_fig(file_name, fig)
ut.print_time('visualization finished')
def train(args):
"""Training helper."""
if not args.model.lower() in ['cbow', 'skipgram']:
logging.error('Unsupported model %s.', args.model)
sys.exit(1)
if args.data.lower() == 'toy':
data = mx.gluon.data.SimpleDataset(nlp.data.Text8(segment='train')[:2])
data, vocab, idx_to_counts = preprocess_dataset(
data, max_vocab_size=args.max_vocab_size)
elif args.data.lower() == 'text8':
data = nlp.data.Text8(segment='train')
data, vocab, idx_to_counts = preprocess_dataset(
data, max_vocab_size=args.max_vocab_size)
elif args.data.lower() == 'fil9':
data = nlp.data.Fil9(max_sentence_length=10000)
data, vocab, idx_to_counts = preprocess_dataset(
data, max_vocab_size=args.max_vocab_size)
elif args.data.lower() == 'wiki':
data, vocab, idx_to_counts = wiki(args.wiki_root, args.wiki_date,
args.wiki_language,
args.max_vocab_size)
if args.ngram_buckets > 0:
data, batchify_fn, subword_function = transform_data_fasttext(
data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow',
ngram_buckets=args.ngram_buckets, ngrams=args.ngrams,
batch_size=args.batch_size, window_size=args.window,
frequent_token_subsampling=args.frequent_token_subsampling)
else:
subword_function = None
data, batchify_fn = transform_data_word2vec(
data, vocab, idx_to_counts, cbow=args.model.lower() == 'cbow',
batch_size=args.batch_size, window_size=args.window,
frequent_token_subsampling=args.frequent_token_subsampling)
num_tokens = float(sum(idx_to_counts))
model = CBOW if args.model.lower() == 'cbow' else SG
embedding = model(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
batch_size=args.batch_size, num_negatives=args.negative,
negatives_weights=mx.nd.array(idx_to_counts),
subword_function=subword_function)
context = get_context(args)
embedding.initialize(ctx=context)
if not args.no_hybridize:
embedding.hybridize(static_alloc=True, static_shape=True)
optimizer_kwargs = dict(learning_rate=args.lr)
try:
trainer = mx.gluon.Trainer(embedding.collect_params(), args.optimizer,
optimizer_kwargs)
except ValueError as e:
if args.optimizer == 'groupadagrad':
logging.warning('MXNet <= v1.3 does not contain '
'GroupAdaGrad support. Falling back to AdaGrad')
trainer = mx.gluon.Trainer(embedding.collect_params(), 'adagrad',
optimizer_kwargs)
else:
raise e
try:
if args.no_prefetch_batch:
data = data.transform(batchify_fn)
else:
from executors import LazyThreadPoolExecutor
num_cpu = len(os.sched_getaffinity(0))
ex = LazyThreadPoolExecutor(num_cpu)
except (ImportError, SyntaxError, AttributeError):
# Py2 - no async prefetching is supported
logging.warning(
'Asynchronous batch prefetching is not supported on Python 2. '
'Consider upgrading to Python 3 for improved performance.')
data = data.transform(batchify_fn)
num_update = 0
prefetched_iters = []
for _ in range(min(args.num_prefetch_epoch, args.epochs)):
prefetched_iters.append(iter(data))
for epoch in range(args.epochs):
if epoch + len(prefetched_iters) < args.epochs:
prefetched_iters.append(iter(data))
data_iter = prefetched_iters.pop(0)
try:
batches = ex.map(batchify_fn, data_iter)
except NameError: # Py 2 or batch prefetching disabled
batches = data_iter
# Logging variables
log_wc = 0
log_start_time = time.time()
log_avg_loss = 0
for i, batch in enumerate(batches):
ctx = context[i % len(context)]
batch = [array.as_in_context(ctx) for array in batch]
with mx.autograd.record():
loss = embedding(*batch)
loss.backward()
num_update += loss.shape[0]
if len(context) == 1 or (i + 1) % len(context) == 0:
trainer.step(batch_size=1)
# Logging
log_wc += loss.shape[0]
log_avg_loss += loss.mean().as_in_context(context[0])
if (i + 1) % args.log_interval == 0:
# Forces waiting for computation by computing loss value
log_avg_loss = log_avg_loss.asscalar() / args.log_interval
wps = log_wc / (time.time() - log_start_time)
# Due to subsampling, the overall number of batches is an upper
# bound
num_batches = num_tokens // args.batch_size
if args.model.lower() == 'skipgram':
num_batches = (num_tokens * args.window * 2) // args.batch_size
else:
num_batches = num_tokens // args.batch_size
logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
'throughput={:.2f}K wps, wc={:.2f}K'.format(
epoch, i + 1, num_batches, log_avg_loss,
wps / 1000, log_wc / 1000))
log_start_time = time.time()
log_avg_loss = 0
log_wc = 0
if args.eval_interval and (i + 1) % args.eval_interval == 0:
with print_time('mx.nd.waitall()'):
mx.nd.waitall()
with print_time('evaluate'):
evaluate(args, embedding, vocab, num_update)
# Evaluate
with print_time('mx.nd.waitall()'):
mx.nd.waitall()
with print_time('evaluate'):
evaluate(args, embedding, vocab, num_update,
eval_analogy=not args.no_eval_analogy)
# Save params
with print_time('save parameters'):
embedding.save_parameters(os.path.join(args.logdir, 'embedding.params'))
def __call__(self):
self.data.update_before_computing_gradients()
g_st = time.time()
self.compute_gradients()
g_ed = time.time()
self.data.update_before_computing_natural_gradients()
r_st = time.time()
rvals = self.compute_natural_gradients()
r_ed = time.time()
self.data.update_before_evaluation()
e_st = time.time()
old_cost = self.compute_old_cost()
new_cost, error = self.compute_new_cost(self.lr)
rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr,
rvals[5]*rvals[6])
if self.state['adapt'] == 3:
odamp = self.damping.get_value()
if rvals[1] < 5 and odamp > 1e-5:
self.damping.set_value(odamp * 2. /3.)
if rvals[1] > 30 and odamp < 1024:
self.damping.set_value(odamp * 3/ 2.)
if self.state['adapt'] == 1:
if rho < .25 and self.damping.get_value() > 0:
self.damping.set_value(numpy.float32(
self.damping.get_value() * 3. / 2.))
elif rho < .25:
self.damping.set_value(numpy.float32(1e-5))
elif rho > .75 and self.damping.get_value() > 1e-4:
self.damping.set_value(numpy.float32(
self.damping.get_value() * 2. / 3.))
#elif rho > .75:
# self.damping.set_value(numpy.float32(0.))
tmp_cost = new_cost
if self.state['adapt'] == 4:
if self.step > self.state['adapt_start']:
if self.step % self.state['adapt_change'] == 0:
self.damping.set_value(
self.damping.get_value() *
self.state['adapt_decrease'])
if self.state['lr_adapt'] == 1:
if self.step > self.state['lr_adapt_start']:
if self.step % self.state['lr_adapt_change'] == 0:
self.lr = self.lr * self.state['lr_adapt_decrease']
elif self.state['lr_adapt'] == 2:
if self.step > self.state['lr_adapt_start']:
self.lr = self.state['lr0'] /\
(1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta'])
self.state['lr'] = float(self.lr)
if self.step % self.state['trainFreq'] == 0:
new_cost, old_cost, error = self.compute_new_cost_all(self.lr)
if new_cost > self.state['btraincost'] * 6:
raise Exception('Variance too large on training cost!')
if self.state['adapt'] == 2:
rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr,
rvals[5]*rvals[6])
if rho < .25 and self.damping.get_value() > 0:
self.damping.set_value(numpy.float32(
self.damping.get_value() * 3. / 2.))
elif rho < .25:
self.damping.set_value(numpy.float32(1e-6))
elif rho > .75 and self.damping.get_value() > 1e-6:
self.damping.set_value(numpy.float32(
self.damping.get_value() * 2. / 3.))
elif rho > .75:
self.damping.set_value(numpy.float32(0.))
self.__new_cost = new_cost
self.__error = error
e_ed = time.time()
print 'Minres: %s' % self.msgs[rvals[0]], \
'# iters %04d' % rvals[1], \
'relative error residuals %.4g' % rvals[2], \
'Anorm', rvals[3], 'Acond', rvals[4]
msg = ('.. iter %4d cost %.3g (%.3g), error %.3g step_size %.3g '
'rho %.3g damping %.4g '
'r_g %.3g '
'ord0_norm %.3g '
'norm grad %.3g '
'norm nat grad %.3g '
'angle %.3g '
'time [grad] %s,'
'[riemann grad] %s,'
'[updates param] %s,'
'whole time %s')
print msg % (
self.step,
new_cost,
tmp_cost,
error,
self.lr,
rho,
self.damping.get_value(),
r_g,
rvals[7],
rvals[5],
rvals[6],
angle,
print_time(g_ed - g_st),
print_time(r_ed - r_st),
print_time(e_ed - e_st),
print_time(time.time() - self.step_timer))
self.step_timer = time.time()
else:
new_cost = self.__new_cost
error = self.__error
self.old_cost = new_cost
self.update_params(self.lr)
e_ed = time.time()
self.step += 1
ret = {
'cost': float(new_cost),
'error': float(error),
'time_grads': float(g_ed - g_st),
'time_metric': float(r_ed - r_st),
'time_eval': float(e_ed - e_st),
'minres_flag': rvals[0],
'minres_iters': rvals[1],
'minres_relres': rvals[2],
'minres_Anorm': rvals[3],
'minres_Acond': rvals[4],
'norm_ord0': rvals[7],
'norm_grad':rvals[5],
'norm_nat': rvals[6],
'grad_angle' : float(angle),
'lr': self.lr,
'r_g': float(r_g),
'icost' : float(tmp_cost),
'damping': self.damping.get_value(),
'rho': numpy.float32(rho)}
return ret
def new_iter_batch(qlts, num, data_type):
n = 0
n_cores = len(qlts)
sleep_t = 0
cache_key_list = cache_data_dict[data_type].keys()
random.shuffle(cache_key_list)
while n < num:
empty = 0
for qlt in qlts:
batch_queue, lock, i = qlt
if len(batch_queue) == 0:
empty += 1
continue
if lock.acquire(False):
print('[Main th] Slept for {} secs.'.format(sleep_t))
# sleep_t = 0
# print('Taking batches from batch_queue {}'.format(i))
# bts = [bt for bt in batch_queue]
# batch_queue.clear()
# del batch_queue[:]
bts = []
while len(batch_queue) > 0:
bts.append(batch_queue.popleft())
print('[Main th] Got {} batches from Queue {}.\t{}'.format(len(bts), i, utils.print_time()))
lock.release()
lg = get_batch_length(bts[0])
st = time.time()
n_elem = 0
for bt in bts:
if type(bt) != type('end'):
# ti = time.time()
yield bt
n_elem += 1
else:
n += 1
print('[Main th] Finished {}: {}*{}.\t{:.3f} secs.'.format(n, lg, n_elem, time.time()-st))
n_elem = 0
sys.stdout.flush()
del bts[:]
ng = gc.collect()
# print('{} garbages.'.format(ng))
del gc.garbage[:]
else:
empty += 1
if len(cache_key_list) > 0:
empty = 0
dt_n = cache_key_list.pop()
bts = cache_data_dict[data_type][dt_n]
print('[Main th] Got {} batches from Cache {}_{}.\t{}'.format(len(bts), data_type, dt_n, utils.print_time()))
random.shuffle(bts)
lg = get_batch_length(bts[0])
st = time.time()
n_elem = 0
for bt in bts:
yield bt
n_elem += 1
n += 1
print('[Main th] Finished {}: {}*{}.\t{:.3f} secs.'.format(n, lg, n_elem, time.time()-st))
n_elem = 0
sys.stdout.flush()
if empty == n_cores:
# print('All empty or unavailable.')
time.sleep(2)
sleep_t += 2
print('[Main th] Totally slept {} secs.'.format(sleep_t))
def transform_data_fasttext(data, vocab, idx_to_counts, cbow, ngram_buckets,
ngrams, batch_size, window_size,
frequent_token_subsampling=1E-4, dtype='float32',
index_dtype='int64'):
"""Transform a DataStream of coded DataSets to a DataStream of batches.
Parameters
----------
data : gluonnlp.data.DataStream
DataStream where each sample is a valid input to
gluonnlp.data.EmbeddingCenterContextBatchify.
vocab : gluonnlp.Vocab
Vocabulary containing all tokens whose indices occur in data. For each
token, it's associated subwords will be computed and used for
constructing the batches. No subwords are used if ngram_buckets is 0.
idx_to_counts : list of int
List of integers such that idx_to_counts[idx] represents the count of
vocab.idx_to_token[idx] in the underlying dataset. The count
information is used to subsample frequent words in the dataset.
Each token is independently dropped with probability 1 - sqrt(t /
(count / sum_counts)) where t is the hyperparameter
frequent_token_subsampling.
cbow : boolean
If True, batches for CBOW are returned.
ngram_buckets : int
Number of hash buckets to consider for the fastText
nlp.vocab.NGramHashes subword function.
ngrams : list of int
For each integer n in the list, all ngrams of length n will be
considered by the nlp.vocab.NGramHashes subword function.
batch_size : int
The returned data stream iterates over batches of batch_size.
window_size : int
The context window size for
gluonnlp.data.EmbeddingCenterContextBatchify.
frequent_token_subsampling : float
Hyperparameter for subsampling. See idx_to_counts above for more
information.
dtype : str or np.dtype, default 'float32'
Data type of data array.
index_dtype : str or np.dtype, default 'int64'
Data type of index arrays.
Returns
-------
gluonnlp.data.DataStream
Stream over batches. Each returned element is a list corresponding to
the arguments for the forward pass of model.SG or model.CBOW
respectively based on if cbow is False or True. If ngarm_buckets > 0,
the returned sample will contain ngrams. Both model.SG or model.CBOW
will handle them correctly as long as they are initialized with the
subword_function returned as second argument by this function (see
below).
gluonnlp.vocab.NGramHashes
The subword_function used for obtaining the subwords in the returned
batches.
"""
if ngram_buckets <= 0:
raise ValueError('Invalid ngram_buckets. Use Word2Vec training '
'pipeline if not interested in ngrams.')
sum_counts = float(sum(idx_to_counts))
idx_to_pdiscard = [
1 - math.sqrt(frequent_token_subsampling / (count / sum_counts))
for count in idx_to_counts]
def subsample(shard):
return [[
t for t, r in zip(sentence,
np.random.uniform(0, 1, size=len(sentence)))
if r > idx_to_pdiscard[t]] for sentence in shard]
data = data.transform(subsample)
batchify = nlp.data.batchify.EmbeddingCenterContextBatchify(
batch_size=batch_size, window_size=window_size, cbow=cbow,
weight_dtype=dtype, index_dtype=index_dtype)
data = data.transform(batchify)
with print_time('prepare subwords'):
subword_function = nlp.vocab.create_subword_function(
'NGramHashes', ngrams=ngrams, num_subwords=ngram_buckets)
# Store subword indices for all words in vocabulary
idx_to_subwordidxs = list(subword_function(vocab.idx_to_token))
subwordidxs = np.concatenate(idx_to_subwordidxs)
subwordidxsptr = np.cumsum([
len(subwordidxs) for subwordidxs in idx_to_subwordidxs])
subwordidxsptr = np.concatenate([
np.zeros(1, dtype=np.int64), subwordidxsptr])
if cbow:
subword_lookup = functools.partial(
cbow_lookup, subwordidxs=subwordidxs,
subwordidxsptr=subwordidxsptr, offset=len(vocab))
else:
subword_lookup = functools.partial(
skipgram_lookup, subwordidxs=subwordidxs,
subwordidxsptr=subwordidxsptr, offset=len(vocab))
max_subwordidxs_len = max(len(s) for s in idx_to_subwordidxs)
if max_subwordidxs_len > 500:
warnings.warn(
'The word with largest number of subwords '
'has {} subwords, suggesting there are '
'some noisy words in your vocabulary. '
'You should filter out very long words '
'to avoid memory issues.'.format(max_subwordidxs_len))
data = UnchainStream(data)
if cbow:
batchify_fn = cbow_fasttext_batch
else:
batchify_fn = skipgram_fasttext_batch
batchify_fn = functools.partial(
batchify_fn, num_tokens=len(vocab) + len(subword_function),
subword_lookup=subword_lookup, dtype=dtype, index_dtype=index_dtype)
return data, batchify_fn, subword_function
def main(_=None, weight_init=tf.random_normal, activation_f=tf.nn.sigmoid, data_min=0, data_scale=1.0, epochs=50,
learning_rate=0.01, prefix=None):
tf.reset_default_graph()
input_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1])
output_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1])
# Grab the data as numpy arrays.
train_input, train_output = data_utils.mnist(training=True)
test_input, test_output = data_utils.mnist(training=False)
train_set = ut.mnist_select_n_classes(train_input, train_output, NUM_CLASSES, min=data_min, scale=data_scale)
test_set = ut.mnist_select_n_classes(test_input, test_output, NUM_CLASSES, min=data_min, scale=data_scale)
train_input, train_output = train_set[0], train_set[0]
test_input, test_output = test_set[0], test_set[0]
ut.print_info('train (min, max): (%f, %f)' % (np.min(train_set[0]), np.max(train_set[0])))
visual_inputs, visual_output = train_set[0][0:BATCH_SIZE], train_set[0][0:BATCH_SIZE]
epoch_reconstruction = []
EPOCH_SIZE = len(train_input) // BATCH_SIZE
TEST_SIZE = len(test_input) // BATCH_SIZE
assert_model(input_placeholder, output_placeholder, test_input, test_output, train_input, train_output, visual_inputs, visual_output)
with pt.defaults_scope(activation_fn=activation_f,
# batch_normalize=True,
# learned_moments_update_rate=0.0003,
# variance_epsilon=0.001,
# scale_after_normalization=True
):
with pt.defaults_scope(phase=pt.Phase.train):
with tf.variable_scope("model") as scope:
output_tensor = decoder(encoder(input_placeholder), weight_init=weight_init)
pretty_loss = loss(output_tensor, output_placeholder)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train = pt.apply_optimizer(optimizer, losses=[pretty_loss])
init = tf.initialize_all_variables()
runner = pt.train.Runner(save_path=FLAGS.save_path)
best_q = 100000
with tf.Session() as sess:
sess.run(init)
for epoch in xrange(epochs):
# Shuffle the training data.
additional_info = ''
if epoch % np.ceil(epochs / 40.0) == 0 or epoch + 1 == epochs:
reconstruct, loss_value = sess.run([output_tensor, pretty_loss], {input_placeholder: visual_inputs, output_placeholder: visual_output})
epoch_reconstruction.append(reconstruct)
additional_info += 'epoch:%d (min, max): (%f %f)' %(epoch, np.min(reconstruct), np.max(reconstruct))
train_input, train_output = data_utils.permute_data(
(train_input, train_output))
runner.train_model(
train,
pretty_loss,
EPOCH_SIZE,
feed_vars=(input_placeholder, output_placeholder),
feed_data=pt.train.feed_numpy(BATCH_SIZE, train_input, train_output),
print_every=None
)
accuracy = runner.evaluate_model(
pretty_loss,
TEST_SIZE,
feed_vars=(input_placeholder, output_placeholder),
feed_data=pt.train.feed_numpy(BATCH_SIZE, test_input, test_output))
ut.print_time('Accuracy after %2d/%d epoch %.2f; %s' % (epoch + 1, epochs, accuracy, additional_info))
if best_q > accuracy:
best_q = accuracy
save_params = {'suf': 'mn_basic', 'act': activation_f, 'e': epochs, 'opt': optimizer, 'lr': learning_rate,
'init': weight_init, 'acu': int(best_q), 'bs': BATCH_SIZE, 'h': HIDDEN_0_SIZE, 'i':prefix}
ut.reconstruct_images_epochs(np.asarray(epoch_reconstruction), visual_output, save_params=save_params)
ut.print_time('Best Quality: %f for %s' % (best_q, ut.to_file_name(save_params)))
ut.reset_start_time()
return best_q
def process_batches(batch_queue, network, exec_func, func):
print('[Queue {}] Start {} {} batches.\t{}'.format(idx, dt_type, len(batch_queue), utils.print_time()))
lg = get_batch_length(batch_queue[0])
st = time.time()
n_elem = 0
n = shared_param.finished_num
if hasattr(shared_param, 'model'):
lasagne.layers.set_all_param_values(network, shared_param.model)
for bt in batch_queue:
if type(bt) != type('end'):
exec_func(bt, func, shared_param)
n_elem += 1
else:
n += 1
print('[Queue {}] Finished {}: {}*{}.\t{:.3f} secs.'.format(idx, n, lg, n_elem, time.time()-st))
n_elem = 0
sys.stdout.flush()
batch_queue.clear()
shared_param.finished_num = n
shared_param.model = lasagne.layers.get_all_param_values(network)
def main(_=None, weight_init=None, activation_f=tf.nn.sigmoid, data_min=0, data_scale=1.0, epochs=3,learning_rate=None):
tf.reset_default_graph()
input_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 2])
output_placeholder = tf.placeholder(tf.float32, [BATCH_SIZE, 28, 28, 1])
# Grab the data as numpy arrays.
train_input, train_output = data_utils.mnist(training=True)
test_input, test_output = data_utils.mnist(training=False)
train_set = ut.mnist_select_n_classes(train_input, train_output, NUM_CLASSES, min=data_min, scale=data_scale)
test_set = ut.mnist_select_n_classes(test_input, test_output, NUM_CLASSES, min=data_min, scale=data_scale)
train_input, train_output = train_set[1], train_set[0]
test_input, test_output = test_set[1], test_set[0]
ut.print_info('train (min, max): (%f, %f)' % (np.min(train_set[0]), np.max(train_set[0])))
visual_inputs, visual_output = train_set[1][0:BATCH_SIZE], train_set[0][0:BATCH_SIZE]
epoch_reconstruction = []
EPOCH_SIZE = len(train_input) // BATCH_SIZE
TEST_SIZE = len(test_input) // BATCH_SIZE
ut.print_info('train: %s' % str(train_input.shape))
ut.print_info('test: %s' % str(test_input.shape))
ut.print_info('output shape: %s' % str(train_output[0].shape))
assert visual_inputs.shape == input_placeholder.get_shape()
assert len(train_input.shape) == len(input_placeholder.get_shape())
assert len(test_input.shape) == len(input_placeholder.get_shape())
assert visual_output.shape == output_placeholder.get_shape()
assert len(train_output.shape) == len(output_placeholder.get_shape())
assert len(test_output.shape) == len(output_placeholder.get_shape())
with pt.defaults_scope(activation_fn=activation_f,
# batch_normalize=True,
# learned_moments_update_rate=0.0003,
# variance_epsilon=0.001,
# scale_after_normalization=True
):
with pt.defaults_scope(phase=pt.Phase.train):
with tf.variable_scope("model") as scope:
output_tensor = decoder(encoder(input_placeholder), weight_init=weight_init)
pretty_loss = loss(output_tensor, output_placeholder)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
# optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
train = pt.apply_optimizer(optimizer, losses=[pretty_loss])
init = tf.initialize_all_variables()
runner = pt.train.Runner(save_path=FLAGS.save_path)
best_q = 100000
with tf.Session() as sess:
sess.run(init)
for epoch in xrange(epochs):
# Shuffle the training data.
if epoch % np.ceil(epochs / 40.0) == 0 or epoch + 1 == epochs:
reconstruct, loss_value = sess.run([output_tensor, pretty_loss], {input_placeholder: visual_inputs, output_placeholder: visual_output})
epoch_reconstruction.append(reconstruct)
ut.print_info('epoch:%d (min, max): (%f %f)' %(epoch, np.min(reconstruct), np.max(reconstruct)))
train_input, train_output = data_utils.permute_data(
(train_input, train_output))
runner.train_model(
train,
pretty_loss,
EPOCH_SIZE,
feed_vars=(input_placeholder, output_placeholder),
feed_data=pt.train.feed_numpy(BATCH_SIZE, train_input, train_output)
)
accuracy = runner.evaluate_model(
pretty_loss,
TEST_SIZE,
feed_vars=(input_placeholder, output_placeholder),
feed_data=pt.train.feed_numpy(BATCH_SIZE, test_input, test_output))
ut.print_time('Accuracy after %d epoch %g%%' % (
epoch + 1, accuracy * 100))
if best_q > accuracy * 10:
best_q = accuracy * 10
ut.reconstruct_images_epochs(np.asarray(epoch_reconstruction), visual_output,
save_params={'suf':'mn_trivs', 'act':activation_f, 'e':epochs, 'opt':optimizer,
'lr': learning_rate, 'init':weight_init, 'acu': int(best_q)})
def train(args):
"""Training helper."""
vocab, row, col, counts = get_train_data(args)
model = GloVe(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
dropout=args.dropout, x_max=args.x_max, alpha=args.alpha,
weight_initializer=mx.init.Uniform(scale=1 / args.emsize))
context = get_context(args)
model.initialize(ctx=context)
if not args.no_hybridize:
model.hybridize(static_alloc=not args.no_static_alloc)
optimizer_kwargs = dict(learning_rate=args.lr, eps=args.adagrad_eps)
params = list(model.collect_params().values())
try:
trainer = mx.gluon.Trainer(params, 'groupadagrad', optimizer_kwargs)
except ValueError:
logging.warning('MXNet <= v1.3 does not contain '
'GroupAdaGrad support. Falling back to AdaGrad')
trainer = mx.gluon.Trainer(params, 'adagrad', optimizer_kwargs)
index_dtype = 'int32'
if counts.shape[0] >= np.iinfo(np.int32).max:
index_dtype = 'int64'
logging.info('Co-occurrence matrix is large. '
'Using int64 to represent sample indices.')
indices = mx.nd.arange(counts.shape[0], dtype=index_dtype)
for epoch in range(args.epochs):
# Logging variables
log_wc = 0
log_start_time = time.time()
log_avg_loss = 0
mx.nd.shuffle(indices, indices) # inplace shuffle
bs = args.batch_size
num_batches = indices.shape[0] // bs
for i in range(num_batches):
batch_indices = indices[bs * i:bs * (i + 1)]
ctx = context[i % len(context)]
batch_row = row[batch_indices].as_in_context(ctx)
batch_col = col[batch_indices].as_in_context(ctx)
batch_counts = counts[batch_indices].as_in_context(ctx)
with mx.autograd.record():
loss = model(batch_row, batch_col, batch_counts)
loss.backward()
if len(context) == 1 or (i + 1) % len(context) == 0:
trainer.step(batch_size=1)
# Logging
log_wc += loss.shape[0]
log_avg_loss += loss.mean().as_in_context(context[0])
if (i + 1) % args.log_interval == 0:
# Forces waiting for computation by computing loss value
log_avg_loss = log_avg_loss.asscalar() / args.log_interval
wps = log_wc / (time.time() - log_start_time)
logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
'throughput={:.2f}K wps, wc={:.2f}K'.format(
epoch, i + 1, num_batches, log_avg_loss,
wps / 1000, log_wc / 1000))
log_dict = dict(
global_step=epoch * len(indices) + i * args.batch_size,
epoch=epoch, batch=i + 1, loss=log_avg_loss,
wps=wps / 1000)
log(args, log_dict)
log_start_time = time.time()
log_avg_loss = 0
log_wc = 0
if args.eval_interval and (i + 1) % args.eval_interval == 0:
with print_time('mx.nd.waitall()'):
mx.nd.waitall()
with print_time('evaluate'):
evaluate(args, model, vocab, i + num_batches * epoch)
# Evaluate
with print_time('mx.nd.waitall()'):
mx.nd.waitall()
with print_time('evaluate'):
evaluate(args, model, vocab, num_batches * args.epochs,
eval_analogy=not args.no_eval_analogy)
# Save params
with print_time('save parameters'):
model.save_parameters(os.path.join(args.logdir, 'glove.params'))
def __call__(self):
self.data.update_before_computing_gradients()
g_st = time.time()
self.compute_gradients()
g_ed = time.time()
self.data.update_before_computing_natural_gradients()
if self.state['lr_adapt'] == 1:
if self.step > self.state['lr_adapt_start']:
if self.step % self.state['lr_adapt_change'] == 0:
self.lr = self.lr * self.state['lr_adapt_decrease']
elif self.state['lr_adapt'] == 2:
if self.step > self.state['lr_adapt_start']:
self.lr = self.state['lr0'] /\
(1. + float(self.step - self.state['lr_adapt_start'])/self.state['lr_beta'])
self.state['lr'] = float(self.lr)
if self.step % self.state['trainFreq'] == 0:
e_st = time.time()
old_cost = self.compute_old_cost()
new_cost, error = self.compute_new_cost(self.lr)
rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr)
new_cost, error = self.compute_new_cost_all(self.lr)
if new_cost > self.state['btraincost'] * 6:
raise Exception('Variance too large on training cost!')
while (numpy.isnan(new_cost) or
numpy.isinf(new_cost)):
raise Exception('Learning rate too small !')
self.old_cost = new_cost
self.update_params(self.lr)
e_ed = time.time()
self._ocost = old_cost
self._ncost = new_cost
self._error = error
self._eetime = e_ed
self._estime = e_st
msg = ('.. iter %4d cost %.3g, error %.3g step_size %.3g '
'rho %.3g '
'norm grad %.3g '
'time [grad] %s,'
'[updates param] %s,'
'whole time %s'
)
print msg % (
self.step,
new_cost,
error,
self.lr,
rho,
norm_grad,
print_time(g_ed - g_st),
print_time(e_ed - e_st),
print_time(time.time() - self.step_timer) )
self.step_timer = time.time()
else:
old_cost = self._ocost
new_cost = self._ncost
error = self._error
e_ed = self._eetime
e_st = self._estime
rho, norm_grad = self.compute_rho(old_cost, new_cost, self.lr)
self.update_params(self.lr)
self.step += 1
ret = {
'cost': float(new_cost),
'error': float(error),
'time_grads': float(g_ed - g_st),
'time_eval': float(e_ed - e_st),
'norm_grad':norm_grad,
'lr': self.lr,
'rho': numpy.float32(rho)}
return ret
def main(stdfeat, num_epochs=300, param_file=None,
model_file=None, testing_type='8k', model='jy',
frame_level='', continue_train=False, delta=False,
gaussian=True):
# gc.disable()
# Load the dataset
print("Loading data...")
test_only = ( model_file and os.path.isfile(model_file) )
# train_batches, val_batches, test_batches = load_dataset(training_type, testing_type,
# fold, augment, delta, test_only, model_file)
# print('train batches: {}'.format(len(train_batches)))
# print('val batches: {}'.format(len(val_batches)))
# print('test batches: {}'.format(len(test_batches)))
# for bats, idx in my_iterator(train_batches, 100):
# np.save('var/stdfeat/train_'+str(idx), bats)
# for bats, idx in my_iterator(val_batches, 100):
# np.save('var/stdfeat/val_'+str(idx), bats)
# for bats, idx in my_iterator(test_batches, 100):
# np.save('var/stdfeat/test_'+str(idx), bats)
# return
test_batches = get_test_batches(stdfeat)
# build networks and functions
train_fn, val_fn, network = init_process(model, gaussian, delta)
if not test_only or continue_train:
if continue_train and os.path.isfile(model_file):
print('Continue training {}'.format(model_file))
val = np.load(model_file)
lasagne.layers.set_all_param_values(network, val)
print("Starting training...")
lowest_err = 100000.0
highest_acc = 0.0
temp_err_filename = '.temp_err_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \
else '.temp_err_{}'.format(os.path.basename(model_file))
temp_acc_filename = '.temp_acc_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \
else '.temp_acc_{}'.format(os.path.basename(model_file))
n_cores = 5
qlts = []
for i in range(n_cores):
# mgr = Manager()
qlts.append((collections.deque(), threading.Lock(), i))
# pool = Pool(processes=n_cores)
# train_queue, val_queue = mgr.list(), mgr.list()
# train_queue, val_queue = Queue(2000), Queue(1000)
# tr_fp_list = get_fp_list(stdfeat, num_epochs, True)
# tr_fp_list = get_fp_list('train', stdfeat, num_epochs, True)
# va_fp_list = get_fp_list('val', stdfeat, num_epochs, True)
# load_data_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list),
# name='tr_loadfile')
# load_train_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list),
# name='tr_loadfile')
# load_val_proc = Process(target=th_loadfile, args=(val_queue, va_fp_list),
# name='va_loadfile')
# load_data_proc.start()
# load_train_proc.start()
# load_val_proc.start()
tr_num = get_data_num('train', stdfeat)
va_num = get_data_num('val', stdfeat)
sys.stdout.flush()
for epoch in range(num_epochs):
print('epoch {}. {}'.format(epoch, utils.print_time()))
start_time = time.time()
# np.random.shuffle(train_batches)
train_err, no_tr, val_err, val_acc, no_va = 0, 0, 0, 0, 0
out_list, tar_list = [], []
result_map = np.zeros((10,10), dtype=np.int32)
if cache_data and epoch != 0:
np.random.shuffle(all_data_queues['train'])
for batch in all_data_queues['train']:
err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn)
if err is not None:
train_err += err*len(pred)
no_tr += len(pred)
# print('Total number of training data: {}'.format(no_tr))
for batch in all_data_queues['val']:
err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn)
if err is not None:
val_err += err*len(pred)
val_acc += acc*len(pred)
no_va += len(pred)
for i, j in zip(targets, pred):
result_map[i.argmax()][j] += 1
# print('Total number of validation data: {}'.format(no_va))
else:
# pool.map_async(pool_loadfile, gen_pool_fp_list('train', stdfeat, qlts, True))
proc_list = gen_procs(qlts, generate_fp_list('train', stdfeat, True), 'train')
for batch in new_iter_batch(qlts, tr_num):
# for batch in iter_batch(train_queue, 'train', tr_num):
err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn)
if err is not None:
train_err += err*len(pred)
no_tr += len(pred)
# print('Join all procs.')
for proc in proc_list:
proc.join()
print('Total number of training data: {}'.format(no_tr))
# pool.map_async(pool_loadfile, gen_pool_fp_list('val', stdfeat, qlts, False))
proc_list = gen_procs(qlts, generate_fp_list('val', stdfeat, True), 'val')
for batch in new_iter_batch(qlts, va_num):
# for batch in iter_batch(train_queue, 'val', va_num):
# for batch in iter_batch(val_queue, 'val', va_num):
err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn)
# print('out:\n {}'.format(out))
# print('targets:\n {}'.format(targets))
if err is not None:
val_err += err*len(pred)
val_acc += acc*len(pred)
no_va += len(pred)
# print((err, acc, pred_prob))
for i, j in zip(targets, pred):
result_map[i.argmax()][j] += 1
# print('Join all procs.')
for proc in proc_list:
proc.join()
print('Total number of validation data: {}'.format(no_va))
# Print the results for this epoch:
print("Epoch {} of {} took {:.3f}s".format(
epoch + 1, num_epochs, time.time() - start_time))
print(" training loss:\t\t{:.6f}".format(train_err / no_tr))
print(" validation loss:\t\t{:.6f}".format(val_err / no_va))
print(" validation accuracy:\t\t{:.2f} %".format(
val_acc / no_va * 100))
print("Result map: (x: prediction, y: target)")
print(result_map)
# Save the best model
if (val_err / no_va) <= lowest_err:
lowest_err = val_err / no_va
final_err_param = lasagne.layers.get_all_param_values(network)
np.save(temp_err_filename, final_err_param)
if (val_acc / no_va) >= highest_acc:
highest_acc = val_acc / no_va
final_acc_param = lasagne.layers.get_all_param_values(network)
np.save(temp_acc_filename, final_acc_param)
if epoch%25 == 24:
run_test(test_batches, val_fn, testing_type, frame_level, print_prob=True)
sys.stdout.flush()
# load_train_proc.join()
# load_val_proc.join()
# pool.close()
# pool.join()
if model_file:
np.save('model/final/' + model_file, lasagne.layers.get_all_param_values(network))
# return to the best weights
val = np.load(temp_acc_filename)
lasagne.layers.set_all_param_values(network, val)
if model_file:
os.rename(temp_acc_filename, 'model/acc/' + model_file)
os.rename(temp_err_filename, 'model/err/' + model_file)
else:
print('Load weights from file {}'.format(model_file))
val = np.load(model_file)
lasagne.layers.set_all_param_values(network, val)
run_test(test_batches, val_fn, testing_type, frame_level)
def __call__(self):
self.data.update_before_computing_gradients()
g_st = time.time()
new_cost = self.compute_gradients()
g_ed = time.time()
self.data.update_before_computing_natural_gradients()
r_st = time.time()
rvals = self.compute_natural_gradients()
r_ed = time.time()
self.data.update_before_evaluation()
e_st = time.time()
old_cost = self.compute_old_cost()
new_cost, error = self.compute_new_cost(self.lr)
rho, r_g, angle = self.compute_rho(old_cost, new_cost, self.lr,
rvals[5]*rvals[6])
if self.step < 1:
rho = .6
if self.state['adapt'] == 1:
if rho < .25:
self.damping.set_value(numpy.float32(
self.damping.get_value() * self.state['damp_ratio']))
elif rho > .75 and self.damping.get_value() > self.state['mindamp']:
self.damping.set_value(numpy.float32(
self.damping.get_value() / self.state['damp_ratio']))
e_ed = time.time()
if new_cost >= old_cost:
print ('Variance too large on training cost!')
self.damping.set_value(numpy.float32(
self.damping.get_value() + 1.))
else:
self.update_params(self.lr)
print 'Minres: %s' % self.msgs[rvals[0]], \
'# iters %04d' % rvals[1], \
'relative error residuals %.4g' % rvals[2], \
'Anorm', rvals[3], 'Acond', rvals[4]
msg = ('.. iter %4d '
'cost %.4e '
'old_cost %.4e '
'step_size %.1e '
' damping %.1e '
'ord0_norm %.1e '
'norm grad %.1e '
'norm nat grad %.1e '
'angle %.1e '
'rho %.1e '
'time [grad] %s,'
'[riemann grad] %s,'
'[updates param] %s,'
'whole time %s')
print msg % (
self.step,
new_cost,
old_cost,
self.lr,
self.damping.get_value(),
rvals[7],
rvals[5],
rvals[6],
angle,
rho,
print_time(g_ed - g_st),
print_time(r_ed - r_st),
print_time(e_ed - e_st),
print_time(time.time() - self.step_timer))
self.step_timer = time.time()
self.old_cost = new_cost
e_ed = time.time()
self.step += 1
ret = {
'cost': float(new_cost),
'old_cost':float(old_cost),
'error': float(error),
'time_grads': float(g_ed - g_st),
'time_metric': float(r_ed - r_st),
'time_eval': float(e_ed - e_st),
'minres_flag': rvals[0],
'minres_iters': rvals[1],
'minres_relres': rvals[2],
'minres_Anorm': rvals[3],
'minres_Acond': rvals[4],
'norm_ord0': rvals[7],
'norm_grad':rvals[5],
'norm_nat': rvals[6],
'grad_angle' : float(angle),
'rho': float(rvals[-1]),
'lr': self.lr,
#'r_g': float(r_g),
#'icost' : float(tmp_cost),
'damping': self.damping.get_value(),
'rho': numpy.float32(rho)
}
return ret
def main(stdfeat, num_epochs=300, param_file=None,
model_file=None, testing_type='8k', model='jy',
frame_level='', continue_train=False, delta=False,
gaussian=True):
# gc.disable()
# Load the dataset
print("Loading data...")
test_only = ( model_file and os.path.isfile(model_file) )
# train_batches, val_batches, test_batches = load_dataset(training_type, testing_type,
# fold, augment, delta, test_only, model_file)
# print('train batches: {}'.format(len(train_batches)))
# print('val batches: {}'.format(len(val_batches)))
# print('test batches: {}'.format(len(test_batches)))
# for bats, idx in my_iterator(train_batches, 100):
# np.save('var/stdfeat/train_'+str(idx), bats)
# for bats, idx in my_iterator(val_batches, 100):
# np.save('var/stdfeat/val_'+str(idx), bats)
# for bats, idx in my_iterator(test_batches, 100):
# np.save('var/stdfeat/test_'+str(idx), bats)
# return
mgr = Manager()
shared_param = mgr.Namespace()
shared_param.m_type = model
shared_param.gaussian = gaussian
shared_param.delta = delta
model_lock = mgr.Lock()
if not test_only or continue_train:
if continue_train and os.path.isfile(model_file):
print('Continue training {}'.format(model_file))
val = np.load(model_file)
shared_param.model = val
print("Starting training...")
lowest_err = 100000.0
highest_acc = 0.0
temp_err_filename = '.temp_err_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \
else '.temp_err_{}'.format(os.path.basename(model_file))
temp_acc_filename = '.temp_acc_model_{}.npy'.format(int(time.time()*100000)) if model_file == None \
else '.temp_acc_{}'.format(os.path.basename(model_file))
n_cores = 5
qlts = []
# global cache_data_dict
# cache_data_dict = {'train':{}, 'val':{}}
# global cache_lock
# cache_lock = threading.Lock()
# shared_param = mgr.dict({'m_type': model, 'gaussian':gaussian, 'delta':delta,
# 'train_err':0, 'no_tr':0, 'val_acc':0, 'val_err':0, 'no_va':0})
for i in range(n_cores):
# mgr = Manager()
qlts.append((mgr.Queue(200), i))
# pool = Pool(processes=n_cores)
# train_queue, val_queue = mgr.list(), mgr.list()
# train_queue, val_queue = Queue(2000), Queue(1000)
# tr_fp_list = get_fp_list(stdfeat, num_epochs, True)
# tr_fp_list = get_fp_list('train', stdfeat, num_epochs, True)
# va_fp_list = get_fp_list('val', stdfeat, num_epochs, True)
# load_data_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list),
# name='tr_loadfile')
# load_train_proc = Process(target=th_loadfile, args=(train_queue, tr_fp_list),
# name='tr_loadfile')
# load_val_proc = Process(target=th_loadfile, args=(val_queue, va_fp_list),
# name='va_loadfile')
# load_data_proc.start()
# load_train_proc.start()
# load_val_proc.start()
tr_num = get_data_num('train', stdfeat)
va_num = get_data_num('val', stdfeat)
sys.stdout.flush()
for epoch in xrange(num_epochs):
print('epoch {}. {}'.format(epoch+1, utils.print_time()))
start_time = time.time()
# np.random.shuffle(train_batches)
# train_err = 0
# no_tr = 0
shared_param.finished_num = 0
shared_param.train_err = 0
shared_param.no_tr = 0
# pool.map_async(pool_loadfile, gen_pool_fp_list('train', stdfeat, qlts, True))
proc_list = gen_procs(n_cores, model_lock, generate_fp_list('train', stdfeat, True), shared_param, 'train')
# for batch in new_iter_batch(qlts, tr_num, 'train'):
# # for batch in iter_batch(train_queue, 'train', tr_num):
# err, pred, g_pred, pg_pred = train_a_batch(batch, train_fn)
# if err is not None:
# train_err += err*len(pred)
# no_tr += len(pred)
# print('Join all procs.')
for proc in proc_list:
proc.join()
print('Total number of training data: {}'.format(shared_param.no_tr))
shared_param.val_err = 0
shared_param.val_acc = 0
out_list, tar_list = [], []
shared_param.result_map = np.zeros((10,10), dtype=np.int32)
shared_param.no_va = 0
# pool.map_async(pool_loadfile, gen_pool_fp_list('val', stdfeat, qlts, False))
proc_list = gen_procs(n_cores, model_lock, generate_fp_list('val', stdfeat, True), shared_param, 'val')
# for batch in new_iter_batch(qlts, va_num, 'val'):
# # for batch in iter_batch(train_queue, 'val', va_num):
# # for batch in iter_batch(val_queue, 'val', va_num):
# err, acc, pred, pred_prob, g_pred, pg_pred, targets = val_a_batch(batch, val_fn)
# # print('out:\n {}'.format(out))
# # print('targets:\n {}'.format(targets))
# if err is not None:
# val_err += err*len(pred)
# val_acc += acc*len(pred)
# no_va += len(pred)
# # print((err, acc, pred_prob))
# for i, j in zip(targets, pred):
# result_map[i.argmax()][j] += 1
# print('Join all procs.')
for proc in proc_list:
proc.join()
print('Total number of validation data: {}'.format(shared_param.no_va))
# Print the results for this epoch:
train_err, no_tr, result_map = shared_param.train_err, shared_param.no_tr, shared_param.result_map
val_acc, val_err, no_va = shared_param.val_acc, shared_param.val_err, shared_param.no_va
print("Epoch {} of {} took {:.3f}s".format(
epoch + 1, num_epochs, time.time() - start_time))
print(" training loss:\t\t{:.6f}".format(train_err / no_tr))
print(" validation loss:\t\t{:.6f}".format(val_err / no_va))
print(" validation accuracy:\t\t{:.2f} %".format(
val_acc / no_va * 100))
print("Result map: (x: prediction, y: target)")
print(result_map)
# Save the best model
if (val_err / no_va) <= lowest_err:
lowest_err = val_err / no_va
final_err_param = shared_param.model
np.save(temp_err_filename, final_err_param)
if (val_acc / no_va) >= highest_acc:
highest_acc = val_acc / no_va
final_acc_param = shared_param.model
np.save(temp_acc_filename, final_acc_param)
if epoch%25 == 24:
gen_test_proc(shared_param, stdfeat, testing_type, frame_level, print_prob=True)
sys.stdout.flush()
# load_train_proc.join()
# load_val_proc.join()
# pool.close()
# pool.join()
if model_file:
np.save('model/final/' + model_file, shared_param.model)
# return to the best weights
val = np.load(temp_acc_filename)
shared_param.model = val
if model_file:
os.rename(temp_acc_filename, 'model/acc/' + model_file)
os.rename(temp_err_filename, 'model/err/' + model_file)
else:
print('Load weights from file {}'.format(model_file))
val = np.load(model_file)
shared_param.model = val
gen_test_proc(shared_param, stdfeat, testing_type, frame_level)
