def fixture_bi_lstm_model():
check_data()
config = load_config()
conf = Namespace(embeddings=Path(config['sentences.wordEmbeddings']),
chars_file=Path(config['sentences.charsFile']),
hparams_file=Path(config['sentences.hparamsFile']),
model_file=Path(config['sentences.modelFile']))
proc = create_processor(conf)
yield proc
proc.close()
def fixture_bi_lstm_model():
check_data()
conf = Namespace(chars_file=None,
words_file=None,
model_hparams=None,
model_file=None,
torch_device='cpu')
proc = bi_lstm.create_processor(conf)
yield proc
proc.close()
def create_processor(conf):
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
logging.basicConfig(level=logging.INFO)
check_data(conf.download_data)
config = load_config()
if conf.embeddings is None:
conf.embeddings = Path(config['sentences.wordEmbeddings'])
if conf.chars_file is None:
conf.chars_file = Path(config['sentences.charsFile'])
if conf.hparams_file is None:
conf.hparams_file = Path(config['sentences.hparamsFile'])
if conf.model_file is None:
conf.model_file = Path(config['sentences.modelFile'])
if conf.torch_device is not None:
device = conf.torch_device
else:
device = "cpu" if conf.force_cpu or not torch.cuda.is_available(
) else "cuda"
device = torch.device(device)
logger.info('Using torch device: "{}"'.format(repr(device)))
logger.info('Loading hparams from: {}'.format(conf.hparams_file))
with conf.hparams_file.open('r') as f:
d = yaml.load(f, Loader)
class Hparams:
pass
hparams = Hparams()
hparams.__dict__.update(d)
logger.info('Loading word embeddings from: "{}"'.format(conf.embeddings))
words, vectors = load_vectors(conf.embeddings)
vectors = np.array(vectors)
logger.info('Loading characters from: {}'.format(conf.chars_file))
char_mapping = load_char_mapping(conf.chars_file)
input_mapping = InputMapping(char_mapping, words, hparams.word_length)
model = BiLSTM(hparams, n_chars(char_mapping), vectors)
model.eval()
model.to(device=device)
model.share_memory()
logger.info('Loading model weights from: {}'.format(conf.model_file))
with conf.model_file.open('rb') as f:
state_dict = torch.load(f)
model.load_state_dict(state_dict)
torch.multiprocessing.set_start_method('fork')
processor = SentenceProcessor(input_mapping, model, device)
return processor
def processor(conf):
logging.basicConfig(level=logging.INFO)
check_data(conf.download_data)
proc = create_processor(conf)
run_processor(proc, namespace=conf)
def processor(conf):
check_data(conf.download_data)
proc = create_processor(conf)
run_processor(proc, namespace=conf)