def main():
args = parse_args()
state = getattr(experiments.nmt, args.proto)()
if args.state:
if args.state.endswith(".py"):
state.update(eval(open(args.state).read()))
else:
with open(args.state) as src:
state.update(cPickle.load(src))
for change in args.changes:
state.update(eval("dict({})".format(change)))
logging.basicConfig(level=getattr(logging, state['level']), format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
logger.debug("State:\n{}".format(pprint.pformat(state)))
rng = numpy.random.RandomState(state['seed'])
if args.proto == 'prototype_ntm_state' or args.proto == 'prototype_ntmencdec_state':
print 'Neural Turing Machine'
enc_dec = NTMEncoderDecoder(state, rng, args.skip_init)
else:
enc_dec = RNNEncoderDecoder(state, rng, args.skip_init)
enc_dec.build()
lm_model = enc_dec.create_lm_model()
logger.debug("Load data")
train_data = get_batch_iterator(state)
logger.debug("Compile trainer")
algo = eval(state['algo'])(lm_model, state, train_data)
logger.debug("Run training")
main = MainLoop(train_data, None, None, lm_model, algo, state, None,
reset=state['reset'],
hooks=[RandomSamplePrinter(state, lm_model, train_data)]
if state['hookFreq'] >= 0
else None)
if state['reload']:
main.load()
if state['loopIters'] > 0:
main.main()
def main():
args = parse_args()
state = prototype_state()
with open(args.state) as src:
state.update(cPickle.load(src))
state.update(eval("dict({})".format(args.changes)))
logging.basicConfig(level=getattr(logging, state['level']), format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
rng = numpy.random.RandomState(state['seed'])
enc_dec = NTMEncoderDecoder(state, rng, skip_init=True)
enc_dec.build()
lm_model = enc_dec.create_lm_model()
lm_model.load(args.model_path)
indx_word = cPickle.load(open(state['word_indx'],'rb'))
sampler = None
beam_search = None
if args.beam_search:
beam_search = BeamSearch(enc_dec)
beam_search.compile()
else:
sampler = enc_dec.create_sampler(many_samples=True)
idict_src = cPickle.load(open(state['indx_word'],'r'))
if args.source and args.trans:
# Actually only beam search is currently supported here
assert beam_search
assert args.beam_size
fsrc = open(args.source, 'r')
ftrans = open(args.trans, 'w')
start_time = time.time()
n_samples = args.beam_size
total_cost = 0.0
logging.debug("Beam size: {}".format(n_samples))
for i, line in enumerate(fsrc):
seqin = line.strip()
print seqin
seq, parsed_in = parse_input(state, indx_word, seqin, idx2word=idict_src)
if args.verbose:
print "Parsed Input:", parsed_in
trans, costs, _ = sample(lm_model, seq, n_samples, sampler=sampler,
beam_search=beam_search, ignore_unk=args.ignore_unk, normalize=args.normalize)
best = numpy.argmin(costs)
print trans[best]
print costs[best]
print >>ftrans, trans[best]
if args.verbose:
print "Translation:", trans[best]
total_cost += costs[best]
if (i + 1) % 100 == 0:
ftrans.flush()
logger.debug("Current speed is {} per sentence".
format((time.time() - start_time) / (i + 1)))
print "Total cost of the translations: {}".format(total_cost)
fsrc.close()
ftrans.close()
else:
while True:
try:
seqin = raw_input('Input Sequence: ')
n_samples = int(raw_input('How many samples? '))
alpha = None
if not args.beam_search:
alpha = float(raw_input('Inverse Temperature? '))
seq,parsed_in = parse_input(state, indx_word, seqin, idx2word=idict_src)
print "Parsed Input:", parsed_in
except Exception:
print "Exception while parsing your input:"
traceback.print_exc()
continue
sample(lm_model, seq, n_samples, sampler=sampler,
beam_search=beam_search,
ignore_unk=args.ignore_unk, normalize=args.normalize,
alpha=alpha, verbose=True)
def main():
args = parse_args()
state = getattr(experiments.nmt, args.proto)()
if args.state:
if args.state.endswith(".py"):
state.update(eval(open(args.state).read()))
else:
with open(args.state) as src:
state.update(cPickle.load(src))
for change in args.changes:
state.update(eval("dict({})".format(change)))
logging.basicConfig(level=getattr(logging, state['level']), format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
logger.debug("State:\n{}".format(pprint.pformat(state)))
rng = numpy.random.RandomState(state['seed'])
if args.proto == 'prototype_ntm_state' or args.proto == 'prototype_ntmencdec_state':
print 'Neural Turing Machine'
enc_dec = NTMEncoderDecoder(state, rng, args.skip_init)
else:
enc_dec = RNNEncoderDecoder(state, rng, args.skip_init)
enc_dec.build()
lm_model = enc_dec.create_lm_model()
#s_enc_dec = RNNEncoderDecoder(state, rng, args.skip_init)
#s_lm_model = s_enc_dec.create_lm_model()
logger.debug("Load data")
train_data = get_batch_iterator(state)
train_data.start(-1)
logger.debug("Compile trainer")
#algo = eval(state['algo'])(lm_model, state, train_data)
#algo()
#train
print '---test training---'
for i in range(1):
batch = train_data.next()
#print batch
x = batch['x']
print x.shape
print x
xs = x[:,78:79]
xsample = x[:,78]
print xs.shape
print xs
print xsample
y = batch['y']
ys = y[:,78:79]
print y.shape
x_mask = batch['x_mask']
xs_mask = x_mask[:,78:79]
y_mask = batch['y_mask']
ys_mask = y_mask[:,78:79]
if not (args.proto == 'prototype_ntm_state' or args.proto == 'prototype_ntmencdec_state'):
print '---search---'
train_outputs = enc_dec.forward_training.rvalss+[enc_dec.predictions.out]
test_train = theano.function(inputs=[enc_dec.x, enc_dec.x_mask, enc_dec.y, enc_dec.y_mask],
outputs=train_outputs)
result = test_train(x,x_mask,y,y_mask)
for i in result:
print i.shape
else:
print '---ntm---'
train_outputs = enc_dec.forward_training.rvalss+[
enc_dec.training_c.out,
enc_dec.forward_training_c.out,
enc_dec.forward_training_m.out,
enc_dec.forward_training_rw.out,
enc_dec.backward_training_c.out,
enc_dec.backward_training_m.out,
enc_dec.backward_training_rw.out,
]
train_outputs = enc_dec.forward_training.rvalss+[\
enc_dec.predictions.out,
enc_dec.training_c.out
]
test_train = theano.function(inputs=[enc_dec.x, enc_dec.x_mask, enc_dec.y, enc_dec.y_mask],
outputs=train_outputs)
result = test_train(x,x_mask,y,y_mask)
for i in result:
print i.shape
#small batch test
print '---small---'
results = test_train(xs,xs_mask,ys,ys_mask)
for i in results:
print i.shape
print '---compare---'
#print result[1][:,4,:,:]
#print results[1][:,0,:,:]
print results[-1].shape
print result[-1][:,78,:]-results[-1][:,0,:]
print numpy.sum(result[-1][:,78,:]-results[-1][:,0,:])
#print numpy.sum(result[0][:,4,:]-results[0][:,0,:])
tmp = copy.deepcopy(result[-1][:,78,:])
tmpm = copy.deepcopy(result[1][:,78,:,:])
#sample
#batch = train_data.next()
#print batch
print '---test sampling---'
x = [7,152,429,731,10239,1127,747,480,30000]
n_samples=10
n_steps=10
T=1
inps = [enc_dec.sampling_x,
enc_dec.n_samples,
enc_dec.n_steps,
enc_dec.T]
#test_sample = theano.function(inputs=[enc_dec.sampling_x],
# outputs=[enc_dec.sample])
test_outputs = [enc_dec.sampling_c,
enc_dec.forward_sampling_c,
enc_dec.forward_sampling_m,
enc_dec.forward_sampling_rw,
enc_dec.backward_sampling_c,
enc_dec.backward_sampling_m,
enc_dec.backward_sampling_rw
]
test_outputs = enc_dec.forward_sampling.rvalss#+[enc_dec.sample,enc_dec.sample_log_prob,enc_dec.sampling_updates]
#test_outputs = [enc_dec.sample,enc_dec.sample_log_prob]
#sample_fn = theano.function(inputs=inps,outputs=test_outputs)
sampler = enc_dec.create_sampler(many_samples=True)
result = sampler(n_samples, n_steps,T,xsample)
#print result
print '---single repr---'
c,m = enc_dec.create_representation_computer()(x)
states = map(lambda x : x[None, :], enc_dec.create_initializers()(c))
#print states
print states[0].shape
print m[-1:].shape
'''
next = enc_dec.create_next_states_computer(c, 0, inputs, m[-1:],*states)
#print next[0]
#print next[1]
print next[0].shape
print next[1].shape
print c
print m
'''
print '---repr compare---'
print c.shape
print m.shape
print c-tmp[0:c.shape[0],:]
print numpy.sum(c-tmp[0:c.shape[0],:],axis=1)
print m-tmpm[0:m.shape[0],:,:]
print numpy.sum(m-tmp[0:m.shape[0],:,:],axis=1)
return
