from word_classifier.data import ClassifierData
import framework.subgraph.losses as losses
import framework.utils.common as utils
from framework.trainer import Trainer, _default_train_iteration_done
from time import time
import model
import os
import shutil
param_file = 'params.py'
params = utils.load_param_file(param_file)
if not utils.get_dict_value(params, 'ignore_negative_data', False):
params['num_classes'] = len(params['keywords']) + 1
else:
params['num_classes'] = len(params['keywords'])
indexer = TextIndexer.from_txt_file(utils.get_dict_value(params, 'vocab_file'))
indexer.add_token('<pad>')
indexer.add_token('unk')
os.makedirs(utils.get_dict_value(params, 'output_location'), exist_ok=True)
indexer.save_vocab_as_pkl(
os.path.join(utils.get_dict_value(params, 'output_location'), 'vocab.pkl'))
shutil.copyfile(
param_file,
os.path.join(utils.get_dict_value(params, 'output_location'), param_file))
params['vocab_size'] = indexer.vocab_size()
training_data = ClassifierData.get_monolingual_training(
base_dir=params['monolingual_dir'], indexer=indexer, params=params)
def on_checkpoint_saved(trainer, params, save_path):
import framework.subgraph.losses as losses
import framework.utils.common as utils
import data
from framework.trainer import Trainer, _default_train_iteration_done
from time import time
import pickle
import model
import os
import shutil
import copy
import numpy as np
param_file = 'params.py'
params = utils.load_param_file(param_file)
params['num_classes'] = len(params['keywords'])+1
indexer = TextIndexer.from_txt_file(utils.get_dict_value(params, 'vocab_file'), max_size=utils.get_dict_value(params,'max_vocab_size',-1))
indexer.add_token('<pad>')
indexer.add_token('unk')
output_indexer = copy.deepcopy(indexer)
output_indexer.add_token('<blank>')
os.makedirs(utils.get_dict_value(params,'output_location'), exist_ok=True)
indexer.save_vocab_as_pkl(os.path.join(utils.get_dict_value(params,'output_location'), 'vocab.pkl'))
files_to_copy = [param_file]
for file in files_to_copy:
shutil.copyfile(file,os.path.join(utils.get_dict_value(params,'output_location'), file))
params['vocab_size'] = indexer.vocab_size()
if 'training_data_dir' in params:
training_data = ClassifierData.get_training_data(base_dir=params['training_data_dir'], indexer=indexer, params=params,
MODEL_NAME = "v0"
params = {
'num_words_before': 10,
'num_words_after': 10,
'embedding_size': 300,
'vocab_size': 100000,
'embedding_device': None,
'batch_size': 128,
'num_classes': 4,
'keywords': ['a', 'an', 'the'],
'mini_batches_between_checkpoint': 100,
'monolingual_dir': '/mnt/work/1-billion-word-language-modeling-benchmark'
}
indexer = TextIndexer.from_txt_file(
os.path.join(params['monolingual_dir'], '1b_word_vocab.txt'))
indexer.add_token('<pad>')
indexer.add_token('unk')
indexer.save_vocab_as_pkl('vocab.pkl')
params['vocab_size'] = indexer.vocab_size()
training_data = ClassifierData.get_monolingual_training(
base_dir=params['monolingual_dir'], indexer=indexer, params=params)
#print(training_data.next_batch(10))
#exit(0)
#print(training_data.next_batch())
def on_checkpoint_saved(trainer, params, save_path):
msg = 'saved checkpoint: ' + save_path
print(msg)