def main(config):
prepare_dirs_loggers(config, os.path.basename(__file__))
corpus_client = corpora.TwitterCorpus(config)
conv_corpus = corpus_client.get_corpus_bow()
train_conv, valid_conv, test_conv, vocab_size = conv_corpus['train'],\
conv_corpus['valid'],\
conv_corpus['test'],\
conv_corpus['vocab_size']
# create data loader that feed the deep models
train_feed = data_loaders.TCDataLoader("Train", train_conv, vocab_size,
config)
valid_feed = data_loaders.TCDataLoader("Valid", valid_conv, vocab_size,
config)
test_feed = data_loaders.TCDataLoader("Test", test_conv, vocab_size,
config)
# for generation
conv_corpus_seq = corpus_client.get_corpus_seq()
train_conv_seq, valid_conv_seq, test_conv_seq = conv_corpus_seq[
'train'], conv_corpus_seq['valid'], conv_corpus_seq['test']
model = conv_models.TDM(corpus_client, config)
if config.use_gpu:
model.cuda()
engine.train(model, train_feed, valid_feed, test_feed, config)
# config.batch_size = 10
train_feed_output = data_loaders.TCDataLoader("Train_Output", train_conv,
vocab_size, config)
test_feed_output = data_loaders.TCDataLoader("Test_Output", test_conv,
vocab_size, config)
valid_feed_output = data_loaders.TCDataLoader("Valid_Output", valid_conv,
vocab_size, config)
if config.output_vis:
with open(os.path.join(config.session_dir, "gen_samples.txt"),
"w") as gen_f:
gen_utils.generate(model,
valid_feed_output,
valid_conv_seq,
config,
num_batch=2,
dest_f=gen_f)
# This tool is independent of the syntool name.
def convert(example, example_class, syntool_name):
example_class.add_array_input(example[0], nolen=True)
example_class.add_int_input(example[1])
example_class.add_int_input(example[2])
example_class.add_int_input(example[3])
example_class.array_output(example[4], nolen=True)
return example_class
if __name__ == "__main__":
random.seed(0)
examples = gen_utils.generate(generate_example)
# Create an output of each type using the convert
# function.
example_sets = gen_utils.build_sets(examples, convert)
# Set up any important sub-fields in any of the tests.
# Need to set an example program for simpl.
example_sets['simpl'].partial_program = """
fun arr, n, m, val, arrout ->
r=0;
p=0;
while(?) {
?;
}
return arrout;
"""
