def __init__(self) -> None:
super().__init__(lazy=False)
# todo: could become args
self._token_indexers = {
'tokens': SingleIdTokenIndexer()
} #'tokens' is the namespace we're using
self._tokenizer = CharacterTokenizer()
def __init__(
self,
namespace: str = "token_characters",
character_tokenizer: CharacterTokenizer = CharacterTokenizer(),
start_tokens: List[str] = None,
end_tokens: List[str] = None,
min_padding_length: int = 0,
token_min_padding_length: int = 0,
) -> None:
super().__init__(token_min_padding_length)
if min_padding_length == 0:
url = "https://github.com/allenai/allennlp/issues/1954"
warnings.warn(
"You are using the default value (0) of `min_padding_length`, "
f"which can cause some subtle bugs (more info see {url}). "
"Strongly recommend to set a value, usually the maximum size "
"of the convolutional layer size when using CnnEncoder.",
UserWarning,
)
self._min_padding_length = min_padding_length
self._namespace = namespace
self._character_tokenizer = character_tokenizer
self._start_tokens = [Token(st) for st in (start_tokens or [])]
self._end_tokens = [Token(et) for et in (end_tokens or [])]
def __init__(
self,
namespace: str = 'token_characters',
character_tokenizer: CharacterTokenizer = CharacterTokenizer()
) -> None:
self._namespace = namespace
self._character_tokenizer = character_tokenizer
def __init__(self,
token_indexers: Dict[str, TokenIndexer] = None,
lazy=False):
super().__init__(lazy=lazy)
self.tokenizer = CharacterTokenizer()
self.token_indexers = token_indexers or {
'tokens': SingleIdTokenIndexer()
}
def __init__(
self,
namespace: str = 'token_characters',
character_tokenizer: CharacterTokenizer = CharacterTokenizer(),
start_tokens: List[str] = None,
end_tokens: List[str] = None) -> None:
self._namespace = namespace
self._character_tokenizer = character_tokenizer
self._start_tokens = [Token(st) for st in (start_tokens or [])]
self._end_tokens = [Token(et) for et in (end_tokens or [])]
def classify(text: str, model: LstmClassifier):
tokenizer = CharacterTokenizer()
token_indexers = {'tokens': SingleIdTokenIndexer()}
tokens = tokenizer.tokenize(text)
instance = Instance({'tokens': TextField(tokens, token_indexers)})
logits = model.forward_on_instances([instance])[0]['logits']
label_id = np.argmax(logits)
label = model.vocab.get_token_from_index(label_id, 'labels')
print('text: {}, label: {}'.format(text, label))
def main():
reader = Seq2SeqDatasetReader(
source_tokenizer=WordTokenizer(),
target_tokenizer=CharacterTokenizer(),
source_token_indexers={'tokens': SingleIdTokenIndexer()},
target_token_indexers={'tokens': SingleIdTokenIndexer(namespace='target_tokens')})
train_dataset = reader.read('data/mt/tatoeba.eng_cmn.train.tsv')
validation_dataset = reader.read('data/mt/tatoeba.eng_cmn.dev.tsv')
vocab = Vocabulary.from_instances(train_dataset + validation_dataset,
min_count={'tokens': 3, 'target_tokens': 3})
en_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EN_EMBEDDING_DIM)
# encoder = PytorchSeq2SeqWrapper(
# torch.nn.LSTM(EN_EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))
encoder = StackedSelfAttentionEncoder(input_dim=EN_EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, projection_dim=128, feedforward_hidden_dim=128, num_layers=1, num_attention_heads=8)
source_embedder = BasicTextFieldEmbedder({"tokens": en_embedding})
# attention = LinearAttention(HIDDEN_DIM, HIDDEN_DIM, activation=Activation.by_name('tanh')())
# attention = BilinearAttention(HIDDEN_DIM, HIDDEN_DIM)
attention = DotProductAttention()
max_decoding_steps = 20 # TODO: make this variable
model = SimpleSeq2Seq(vocab, source_embedder, encoder, max_decoding_steps,
target_embedding_dim=ZH_EMBEDDING_DIM,
target_namespace='target_tokens',
attention=attention,
beam_size=8,
use_bleu=True)
optimizer = optim.Adam(model.parameters())
iterator = BucketIterator(batch_size=32, sorting_keys=[("source_tokens", "num_tokens")])
iterator.index_with(vocab)
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
num_epochs=1,
cuda_device=CUDA_DEVICE)
for i in range(50):
print('Epoch: {}'.format(i))
trainer.train()
predictor = SimpleSeq2SeqPredictor(model, reader)
for instance in itertools.islice(validation_dataset, 10):
print('SOURCE:', instance.fields['source_tokens'].tokens)
print('GOLD:', instance.fields['target_tokens'].tokens)
print('PRED:', predictor.predict_instance(instance)['predicted_tokens'])
def test_count_vocab_items_respects_casing(self):
indexer = TokenCharactersIndexer("characters")
counter = defaultdict(lambda: defaultdict(int))
indexer.count_vocab_items(Token("Hello"), counter)
indexer.count_vocab_items(Token("hello"), counter)
assert counter["characters"] == {"h": 1, "H": 1, "e": 2, "l": 4, "o": 2}
indexer = TokenCharactersIndexer("characters", CharacterTokenizer(lowercase_characters=True))
counter = defaultdict(lambda: defaultdict(int))
indexer.count_vocab_items(Token("Hello"), counter)
indexer.count_vocab_items(Token("hello"), counter)
assert counter["characters"] == {"h": 2, "e": 2, "l": 4, "o": 2}
def read_shakespeare(all_chars: Set[str] = None) -> List[List[Token]]:
tokenizer = CharacterTokenizer()
sentences = []
with open('data/shakespeare/hamlet.txt') as f:
for line in f:
line = line.strip()
if not line:
continue
line = re.sub(' +', ' ', line)
tokens = tokenizer.tokenize(line)
if all_chars:
tokens = [token for token in tokens if token.text in all_chars]
sentences.append(tokens)
return sentences
def get_squad_from_rawfile(squad_file_path, need_char=False):
'''
Args:
squad_file_path -- a raw squad.json path
need_char --
Returns:
dataset -- [], each item is a SquadData object, which contains
passage, question, span_start, span_end. Tokenized
'''
namespace_word = "word2idx"
token_indexers = {
"tokens": SingleIdTokenIndexer(namespace=namespace_word)}
reader = SquadReader(token_indexers=token_indexers)
instances = reader.read(squad_file_path)
dataset = []
for instance in instances:
passage = instance.fields['passage'].tokens
passage = [word.text for word in passage]
question = instance.fields['question'].tokens
question = [word.text for word in question]
span_start = instance.fields['span_start'].sequence_index
span_end = instance.fields['span_end'].sequence_index
item = SquadData(passage, question, span_start, span_end)
meta = instance.fields['metadata'].metadata
item.question_id = meta['question_id']
item.passage_text = meta['original_passage']
item.question_text = meta['question_text']
item.answer_text = meta['answer_texts'][0]
item.passage_token_offsets = meta['token_offsets']
dataset.append(item)
print ("read {} datas from {}".format(len(dataset), squad_file_path))
if need_char == True:
char_tokenizer = CharacterTokenizer()
# add char tokenizers
for squad in dataset:
passage_char = char_tokenizer.batch_tokenize(squad.passage)
question_char = char_tokenizer.batch_tokenize(squad.question)
squad.passage_char = [[ch.text for ch in chars] for chars in passage_char]
squad.question_char = [[ch.text for ch in chars] for chars in question_char]
print ("add passage_char, question_char for each squad object")
return dataset
from allennlp.modules.seq2seq_encoders import PytorchSeq2SeqWrapper, StackedSelfAttentionEncoder
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.predictors import SimpleSeq2SeqPredictor
from allennlp.training.trainer import Trainer
EN_EMBEDDING_DIM = 256
ZH_EMBEDDING_DIM = 256
HIDDEN_DIM = 256
CUDA_DEVICE = 0
#def main():
reader = Seq2SeqDatasetReader(
source_tokenizer=WordTokenizer(),
target_tokenizer=CharacterTokenizer(),
source_token_indexers={'tokens': SingleIdTokenIndexer()},
target_token_indexers={
'tokens': SingleIdTokenIndexer(namespace='target_tokens')
})
train_dataset = reader.read('/.../en_el_train.txt')
validation_dataset = reader.read('/.../en_el_dev.txt')
vocab = Vocabulary.from_instances(train_dataset + validation_dataset,
min_count={
'tokens': 3,
'target_tokens': 3
})
en_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EN_EMBEDDING_DIM)
def main():
###############################################################################################
prepare_global_logging(serialization_dir=args.serialization_dir, file_friendly_logging=False)
#DATA
reader = MathDatasetReader(source_tokenizer=CharacterTokenizer(),
target_tokenizer=CharacterTokenizer(),
source_token_indexers={'tokens': SingleIdTokenIndexer(namespace='tokens')},
target_token_indexers={'tokens': SingleIdTokenIndexer(namespace='tokens')},
target=False,
label=True,
lazy=False)
# train_data = reader.read("../../datasets/math/label-data/train-all")
# val_data = reader.read("../../datasets/math/label-data/interpolate")
val_data = reader.read("./generate_files")
vocab = Vocabulary()
vocab.add_tokens_to_namespace([START_SYMBOL, END_SYMBOL, ' ', '!', "'", '(', ')', '*', '+', ',', '-', '.', '/',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', '<', '=', '>', '?',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b',
'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '}'], namespace='tokens')
vocab.add_tokens_to_namespace(['algebra', 'arithmetic', 'calculus', 'comparison',
'measurement', 'numbers', 'polynomials', 'probability'], namespace='labels')
# MODEL
embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EMBEDDING_DIM)
source_embedder = BasicTextFieldEmbedder({"tokens": embedding})
if args.model == 'lstm':
encoder = PytorchSeq2VecWrapper(torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM,
num_layers=NUM_LAYERS, batch_first=True))
elif args.model == 'cnn':
encoder = CnnEncoder(embedding_dim=EMBEDDING_DIM, num_filters=NUM_FILTERS, output_dim=HIDDEN_DIM)
else:
raise NotImplemented("The classifier model should be LSTM or CNN")
model = TextClassifier(vocab=vocab,
source_text_embedder=source_embedder,
encoder=encoder,
)
model.to(device)
if not Path(args.serialization_dir).exists() or not Path(args.serialization_dir).is_dir():
raise NotImplementedError("The model seems not to exist")
with open(Path(args.serialization_dir) / "best.th", "rb") as model_path:
model_state = torch.load(model_path, map_location=nn_util.device_mapping(-1))
model.load_state_dict(model_state)
model.eval()
predictor = TextClassifierPredictor(model, dataset_reader=reader)
# TEST
correct = 0
total = 0
pbar = tqdm(val_data)
batch_instance = list()
batch_gt = list()
idx_last = 0
for idx, instance in enumerate(pbar):
if idx != (idx_last + BATCH_SIZE):
batch_instance.append(instance)
batch_gt.append(instance.fields["labels"].label) # str
else:
idx_last = idx
outputs = predictor.predict(batch_instance)
for i, output in enumerate(outputs):
if batch_gt[i] == output['predict_labels']:
correct += 1
total += 1
batch_instance = list()
batch_gt = list()
pbar.set_description("correct/total %.3f" % (correct / total))
def main():
###############################################################################################
prepare_global_logging(serialization_dir=args.serialization_dir,
file_friendly_logging=False)
#DATA
reader = MathDatasetReader(source_tokenizer=CharacterTokenizer(),
target_tokenizer=CharacterTokenizer(),
source_token_indexers={
'tokens':
SingleIdTokenIndexer(namespace='tokens')
},
target_token_indexers={
'tokens':
SingleIdTokenIndexer(namespace='tokens')
},
target=False,
label=True,
lazy=True)
train_data = reader.read("../../datasets/math/label-data/train-all")
# val_data = reader.read("../../datasets/math/label-data/interpolate")
vocab = Vocabulary()
vocab.add_tokens_to_namespace([
START_SYMBOL, END_SYMBOL, ' ', '!', "'", '(', ')', '*', '+', ',', '-',
'.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', '<',
'=', '>', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{',
'}'
],
namespace='tokens')
vocab.add_tokens_to_namespace([
'algebra', 'arithmetic', 'calculus', 'comparison', 'measurement',
'numbers', 'polynomials', 'probability'
],
namespace='labels')
# MODEL
embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=EMBEDDING_DIM)
source_embedder = BasicTextFieldEmbedder({"tokens": embedding})
if args.model == 'lstm':
encoder = PytorchSeq2VecWrapper(
torch.nn.LSTM(EMBEDDING_DIM,
HIDDEN_DIM,
num_layers=NUM_LAYERS,
batch_first=True))
elif args.model == 'cnn':
encoder = CnnEncoder(embedding_dim=EMBEDDING_DIM,
num_filters=NUM_FILTERS,
output_dim=HIDDEN_DIM)
else:
raise NotImplemented("The classifier model should be LSTM or CNN")
model = TextClassifier(
vocab=vocab,
source_text_embedder=source_embedder,
encoder=encoder,
)
model.to(device)
optimizer = optim.Adam(model.parameters(),
lr=1e-3,
betas=(0.9, 0.995),
eps=1e-6)
train_iterator = BucketIterator(batch_size=BATCH_SIZE,
max_instances_in_memory=1024,
sorting_keys=[("source_tokens",
"num_tokens")])
train_iterator = MultiprocessIterator(train_iterator, num_workers=16)
train_iterator.index_with(vocab)
val_iterator = BucketIterator(batch_size=BATCH_SIZE,
max_instances_in_memory=1024,
sorting_keys=[("source_tokens", "num_tokens")
])
val_iterator = MultiprocessIterator(val_iterator, num_workers=16)
val_iterator.index_with(vocab)
#pdb.set_trace()
LR_SCHEDULER = {"type": "exponential", "gamma": 0.5, "last_epoch": -1}
lr_scheduler = LearningRateScheduler.from_params(optimizer,
Params(LR_SCHEDULER))
# TRAIN
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=train_iterator,
validation_iterator=None,
train_dataset=train_data,
validation_dataset=None,
patience=None,
shuffle=True,
num_epochs=1,
summary_interval=100,
learning_rate_scheduler=lr_scheduler,
cuda_device=CUDA_DEVICES,
grad_norm=5,
grad_clipping=5,
model_save_interval=600,
serialization_dir=args.serialization_dir,
keep_serialized_model_every_num_seconds=3600,
should_log_parameter_statistics=True,
should_log_learning_rate=True)
trainer.train()
from allennlp.data.tokenizers.character_tokenizer import CharacterTokenizer
from allennlp.data.tokenizers.word_tokenizer import WordTokenizer
from allennlp.data.token_indexers import SingleIdTokenIndexer
data_reader_configs = {
'debug': None,
'ud-eng': None,
'nc_zhen': {
'source_tokenizer': WordTokenizer(),
'target_tokenizer': CharacterTokenizer(),
'source_token_indexers': {
'tokens': SingleIdTokenIndexer()
},
'target_token_indexers': {
'tokens': SingleIdTokenIndexer(namespace='target_tokens')
}
},
'wikitext': {}
}
def get_datareader_configs(dataset_name):
return data_reader_configs.get(dataset_name)
def __init__(self,
namespace=u'token_characters',
character_tokenizer=CharacterTokenizer()):
self._namespace = namespace
self._character_tokenizer = character_tokenizer