valid_data_dir = '/mnt/raid1/billion-word-corpus/1-billion-word-language-modeling-benchmark/heldout-monolingual.tokenized.shuffled/'
save_dir = '/home/ab455/language-model/checkpoints/'
num_words = None
seq_len = 25
batch_size = 256
valid_batch_size = 16 ## Needs to be smaller due to memory issues
embed_size = 128
num_epochs = 20
hidden_size = 256
num_layers = 2
dataset = Dataset(data_dir,num_words)
dataset.set_batch_size(batch_size)
dataset.set_seq_len(seq_len)
dataset.save('./checkpoints_large/')
params = {}
params['vocab_size'] = dataset.vocab_size
params['num_classes'] = dataset.vocab_size
params['batch_size'] = batch_size
params['seq_len'] = seq_len
params['hidden_dim'] = hidden_size
params['num_layers'] = num_layers
params['embed_size'] = embed_size
model = LargeLanguageModel(params)
model.compile()
eval_softmax = 5
for epoch in range(num_epochs):
dataset.set_data_dir(data_dir)
os.makedirs(directoryOutLogs)
num_words = None
seq_len = 25
batch_size = 16
valid_batch_size = 16 ## Needs to be smaller due to memory issues
embed_size = 64
num_epochs = 20
hidden_size = 64
num_layers = 1
dataset = Dataset(data_dir, num_words)
dataset.set_batch_size(batch_size)
dataset.set_seq_len(seq_len)
dataset.save(dataset_specific_info)
params = {}
#take account of the 0 token for padding
params['vocab_size'] = dataset.vocab_size + 1
params['num_classes'] = dataset.vocab_size
params['batch_size'] = batch_size
params['valid_batch_size'] = valid_batch_size
params['seq_len'] = seq_len
params['hidden_dim'] = hidden_size
params['num_layers'] = num_layers
params['embed_size'] = embed_size
params['directoryOutLogs'] = directoryOutLogs
model = LanguageModel(params)
valid_data_dir = '/mnt/raid1/billion-word-corpus/1-billion-word-language-modeling-benchmark/heldout-monolingual.tokenized.shuffled/'
save_dir = '/home/ab455/language-model/checkpoints/'
num_words = None
seq_len = 25
batch_size = 192
valid_batch_size = 16 ## Needs to be smaller due to memory issues
embed_size = 128
num_epochs = 20
hidden_size = 256
num_layers = 1
dataset = Dataset(data_dir, num_words)
dataset.set_batch_size(batch_size)
dataset.set_seq_len(seq_len)
dataset.save('./checkpoints/')
params = {}
params['vocab_size'] = dataset.vocab_size
params['num_classes'] = dataset.vocab_size
params['batch_size'] = batch_size
params['seq_len'] = seq_len
params['hidden_dim'] = hidden_size
params['num_layers'] = num_layers
params['embed_size'] = embed_size
model = LanguageModel(params)
model.compile()
eval_softmax = 5
for epoch in range(num_epochs):
dataset.set_data_dir(data_dir)
