def init_model(char_to_ix, tag_to_ix, START_CHAR_ID, STOP_CHAR_ID,
START_TAG_ID, STOP_TAG_ID):
if args.old_model is not None:
model = torch.load(args.old_model)
else:
if args.char_embeddings is not None:
char_embeddings = utils.read_pretrained_embeddings(
args.char_embeddings, char_to_ix)
EMBEDDING_DIM = char_embeddings.shape[1]
else:
char_embeddings = None
EMBEDDING_DIM = args.char_embeddings_dim
model = BiLSTM_CRF(len(char_to_ix), len(tag_to_ix), START_CHAR_ID,
STOP_CHAR_ID, START_TAG_ID, STOP_TAG_ID,
args.use_bigram, args.hidden_dim, args.dropout,
EMBEDDING_DIM, char_embeddings)
return processor.to_cuda_if_available(model)
output["test_instances"], output["test_vocab"] = read_file(
options.test_data, w2i, t2i, c2i)
# Add special tokens / tags / chars to dicts
w2i[UNK_TAG] = len(w2i)
t2i[START_TAG] = len(t2i)
t2i[END_TAG] = len(t2i)
c2i[UNK_TAG] = len(c2i)
output["w2i"] = w2i
output["t2i"] = t2i
output["c2i"] = c2i
# Read embedding
if options.word_embeddings:
output["word_embeddings"] = read_pretrained_embeddings(
options.word_embeddings, w2i)
make_sure_path_exists(os.path.dirname(options.output))
print('Saving dataset to {}'.format(options.output))
with open(options.output, "wb") as outfile:
pickle.dump(output, outfile)
with codecs.open(os.path.dirname(options.output) + "/words.txt", "w",
"utf-8") as vocabfile:
for word in w2i.keys():
vocabfile.write(word + "\n")
with codecs.open(os.path.dirname(options.output) + "/chars.txt", "w",
"utf-8") as vocabfile:
for char in c2i.keys():
def add_word(word):
if word in w2i:
pass
id = len(w2i)
w2i[word] = id
i2w.append(word)
assert len(w2i) == len(i2w)
assert i2w[id] == word
add_word(START_TAG)
add_word(END_TAG)
if options.no_we:
word_embeddings = None
elif options.word_embeddings is not None:
word_embeddings = utils.read_pretrained_embeddings(options.word_embeddings,
w2i)
else:
word_embeddings = dataset.get(
"word_embeddings") # can be stored within dataset
if options.char_embeddings is not None:
char_embeddings = utils.read_pretrained_embeddings(options.char_embeddings,
c2i)
else:
char_embeddings = dataset.get(
"char_embeddings") # can be stored within dataset
# Tie embeddings up
if options.use_char_rnn and word_embeddings is not None and char_embeddings is not None and options.tie_two_embeddings:
for c, i in c2i.items():
if c not in w2i:
help="use my own lstm")
parser.add_argument("--test", dest="test", action="store_true", help="test")
options = parser.parse_args()
print(options)
data = pickle.load(open(options.data, "rb"))
w2i = data["w2i"]
i2w = utils.to_id_list(w2i)
max_length = data["max_length"]
train_data = data["train_data"]
dev_data = data["dev_data"]
print(len(train_data), len(dev_data), len(w2i), max_length)
if options.word_embeddings is not None:
freeze = False
print("freeze embedding:", freeze)
init_embedding = utils.read_pretrained_embeddings(options.word_embeddings,
w2i, options.embed_dim)
embed = nn.Embedding.from_pretrained(torch.FloatTensor(init_embedding),
freeze=freeze)
else:
embed = nn.Embedding(len(w2i), options.embed_dim)
model = model.Generator(embed,
options.embed_dim,
len(w2i),
options.hidden_size,
num_layers=options.layers,
dropout=options.dropout,
use_API=options.API)
optimizer = torch.optim.Adam(model.parameters(), lr=options.lr)
#optimizer=torch.optim.SGD(model.parameters(), lr = options.lr,momentum=0)
criterion = nn.CrossEntropyLoss()