def read_stacked_data(source_path,
word_alphabet,
char_alphabet,
pos_alphabet,
type_alphabet,
max_size=None,
normalize_digits=True,
prior_order='deep_first'):
data = [[] for _ in _buckets]
max_char_length = [0 for _ in _buckets]
print('Reading data from %s' % source_path)
counter = 0
reader = CoNLLXReader(source_path, word_alphabet, char_alphabet,
pos_alphabet, type_alphabet)
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
while inst is not None and (not max_size or counter < max_size):
counter += 1
if counter % 10000 == 0:
print("reading data: %d" % counter)
inst_size = inst.length()
sent = inst.sentence
for bucket_id, bucket_size in enumerate(_buckets):
if inst_size < bucket_size:
stacked_heads, children, siblings, stacked_types, skip_connect, previous, next = _generate_stack_inputs(
inst.heads, inst.type_ids, prior_order)
data[bucket_id].append([
sent.word_ids, sent.char_id_seqs, inst.pos_ids, inst.heads,
inst.type_ids, stacked_heads, children, siblings,
stacked_types, skip_connect, previous, next
])
max_len = max([len(char_seq) for char_seq in sent.char_seqs])
if max_char_length[bucket_id] < max_len:
max_char_length[bucket_id] = max_len
break
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
reader.close()
print("Total number of data: %d" % counter)
return data, max_char_length
def read_bucketed_data(source_path,
bert_path,
bert_dim,
word_alphabet,
char_alphabet,
pos_alphabet,
type_alphabet,
max_size=None,
normalize_digits=True,
prior_order='inside_out'):
data = [[] for _ in _buckets]
max_char_length = [0 for _ in _buckets]
print('Reading data from %s' % source_path)
counter = 0
reader = CoNLLXReader(source_path, bert_path, bert_dim, word_alphabet,
char_alphabet, pos_alphabet, type_alphabet)
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
while inst is not None and (not max_size or counter < max_size):
counter += 1
if counter % 10000 == 0:
print("reading data: %d" % counter)
inst_size = inst.length()
sent = inst.sentence
for bucket_id, bucket_size in enumerate(_buckets):
if inst_size < bucket_size:
stacked_heads, children, siblings, stacked_types, skip_connect = _generate_stack_inputs(
inst.heads, inst.type_ids, prior_order)
data[bucket_id].append([
sent.word_ids, sent.char_id_seqs, inst.bert_embs,
inst.pos_ids, inst.heads, inst.type_ids, stacked_heads,
children, siblings, stacked_types, skip_connect
])
max_len = max([len(char_seq) for char_seq in sent.char_seqs])
if max_char_length[bucket_id] < max_len:
max_char_length[bucket_id] = max_len
break
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
reader.close()
print("Total number of data: %d" % counter)
bucket_sizes = [len(data[b]) for b in range(len(_buckets))]
data_tensors = []
for bucket_id in range(len(_buckets)):
bucket_size = bucket_sizes[bucket_id]
if bucket_size == 0:
data_tensors.append((1, 1))
continue
bucket_length = _buckets[bucket_id]
char_length = min(MAX_CHAR_LENGTH, max_char_length[bucket_id])
wid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
cid_inputs = np.empty([bucket_size, bucket_length, char_length],
dtype=np.int64)
pid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
hid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
bert_inputs = np.empty([bucket_size, bucket_length, bert_dim],
dtype=np.float32) #BERT
#bert_inputs = np.empty([bucket_size, bucket_length], dtype=np.float32)
tid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
masks_e = np.zeros([bucket_size, bucket_length], dtype=np.float32)
single = np.zeros([bucket_size, bucket_length], dtype=np.int64)
lengths = np.empty(bucket_size, dtype=np.int64)
"""
stack_hid_inputs = np.empty([bucket_size, 2 * bucket_length - 1], dtype=np.int64)
chid_inputs = np.empty([bucket_size, 2 * bucket_length - 1], dtype=np.int64)
ssid_inputs = np.empty([bucket_size, 2 * bucket_length - 1], dtype=np.int64)
stack_tid_inputs = np.empty([bucket_size, 2 * bucket_length - 1], dtype=np.int64)
skip_connect_inputs = np.empty([bucket_size, 2 * bucket_length - 1], dtype=np.int64)
masks_d = np.zeros([bucket_size, 2 * bucket_length - 1], dtype=np.float32)
"""
stack_hid_inputs = np.empty([bucket_size, bucket_length - 1],
dtype=np.int64)
chid_inputs = np.empty([bucket_size, bucket_length - 1],
dtype=np.int64)
ssid_inputs = np.empty([bucket_size, bucket_length - 1],
dtype=np.int64)
stack_tid_inputs = np.empty([bucket_size, bucket_length - 1],
dtype=np.int64)
skip_connect_inputs = np.empty([bucket_size, bucket_length - 1],
dtype=np.int64)
masks_d = np.zeros([bucket_size, bucket_length - 1], dtype=np.float32)
for i, inst in enumerate(data[bucket_id]):
wids, cid_seqs, bert, pids, hids, tids, stack_hids, chids, ssids, stack_tids, skip_ids = inst
inst_size = len(wids)
lengths[i] = inst_size
# word ids
wid_inputs[i, :inst_size] = wids
wid_inputs[i, inst_size:] = PAD_ID_WORD
for c, cids in enumerate(cid_seqs):
cid_inputs[i, c, :len(cids)] = cids
cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
#BERT
bert_inputs[i, :inst_size, :bert_dim] = bert
bert_inputs[i, inst_size:, bert_dim:] = PAD_ID_WORD
# pos ids
pid_inputs[i, :inst_size] = pids
pid_inputs[i, inst_size:] = PAD_ID_TAG
# type ids
tid_inputs[i, :inst_size] = tids
tid_inputs[i, inst_size:] = PAD_ID_TAG
# heads
hid_inputs[i, :inst_size] = hids
hid_inputs[i, inst_size:] = PAD_ID_TAG
# masks_e
masks_e[i, :inst_size] = 1.0
for j, wid in enumerate(wids):
if word_alphabet.is_singleton(wid):
single[i, j] = 1
#inst_size_decoder = 2 * inst_size - 1
inst_size_decoder = inst_size - 1
# stacked heads
stack_hid_inputs[i, :inst_size_decoder] = stack_hids
stack_hid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# children
chid_inputs[i, :inst_size_decoder] = chids
chid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# siblings
ssid_inputs[i, :inst_size_decoder] = ssids
ssid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# stacked types
stack_tid_inputs[i, :inst_size_decoder] = stack_tids
stack_tid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# skip connects
skip_connect_inputs[i, :inst_size_decoder] = skip_ids
skip_connect_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# masks_d
masks_d[i, :inst_size_decoder] = 1.0
words = torch.from_numpy(wid_inputs)
chars = torch.from_numpy(cid_inputs)
berts = torch.from_numpy(bert_inputs)
pos = torch.from_numpy(pid_inputs)
heads = torch.from_numpy(hid_inputs)
types = torch.from_numpy(tid_inputs)
masks_e = torch.from_numpy(masks_e)
single = torch.from_numpy(single)
lengths = torch.from_numpy(lengths)
stacked_heads = torch.from_numpy(stack_hid_inputs)
children = torch.from_numpy(chid_inputs)
siblings = torch.from_numpy(ssid_inputs)
stacked_types = torch.from_numpy(stack_tid_inputs)
skip_connect = torch.from_numpy(skip_connect_inputs)
masks_d = torch.from_numpy(masks_d)
#print(berts.dim())
data_tensor = {
'WORD': words,
'CHAR': chars,
'BERT': berts,
'POS': pos,
'HEAD': heads,
'TYPE': types,
'MASK_ENC': masks_e,
'SINGLE': single,
'LENGTH': lengths,
'STACK_HEAD': stacked_heads,
'CHILD': children,
'SIBLING': siblings,
'STACK_TYPE': stacked_types,
'SKIP_CONNECT': skip_connect,
'MASK_DEC': masks_d
}
data_tensors.append(data_tensor)
return data_tensors, bucket_sizes
def read_bucketed_data(source_path: str,
word_alphabet: Alphabet,
char_alphabet: Alphabet,
pos_alphabet: Alphabet,
type_alphabet: Alphabet,
max_size=None,
normalize_digits=True,
symbolic_root=False,
symbolic_end=False):
data = [[] for _ in _buckets]
max_char_length = [0 for _ in _buckets]
print('Reading data from %s' % source_path)
counter = 0
reader = CoNLLXReader(source_path, word_alphabet, char_alphabet,
pos_alphabet, type_alphabet)
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=symbolic_root,
symbolic_end=symbolic_end)
while inst is not None and (not max_size or counter < max_size):
counter += 1
if counter % 10000 == 0:
print("reading data: %d" % counter)
inst_size = inst.length()
sent = inst.sentence
for bucket_id, bucket_size in enumerate(_buckets):
if inst_size < bucket_size:
data[bucket_id].append([
sent.word_ids, sent.char_id_seqs, inst.pos_ids, inst.heads,
inst.type_ids
])
max_len = max([len(char_seq) for char_seq in sent.char_seqs])
if max_char_length[bucket_id] < max_len:
max_char_length[bucket_id] = max_len
break
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=symbolic_root,
symbolic_end=symbolic_end)
reader.close()
print("Total number of data: %d" % counter)
bucket_sizes = [len(data[b]) for b in range(len(_buckets))]
data_tensors = []
for bucket_id in range(len(_buckets)):
bucket_size = bucket_sizes[bucket_id]
if bucket_size == 0:
data_tensors.append((1, 1))
continue
bucket_length = _buckets[bucket_id]
char_length = min(MAX_CHAR_LENGTH, max_char_length[bucket_id])
wid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
cid_inputs = np.empty([bucket_size, bucket_length, char_length],
dtype=np.int64)
pid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
hid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
tid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
masks = np.zeros([bucket_size, bucket_length], dtype=np.float32)
single = np.zeros([bucket_size, bucket_length], dtype=np.int64)
lengths = np.empty(bucket_size, dtype=np.int64)
for i, inst in enumerate(data[bucket_id]):
wids, cid_seqs, pids, hids, tids = inst
inst_size = len(wids)
lengths[i] = inst_size
# word ids
wid_inputs[i, :inst_size] = wids
wid_inputs[i, inst_size:] = PAD_ID_WORD
for c, cids in enumerate(cid_seqs):
cid_inputs[i, c, :len(cids)] = cids
cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
# pos ids
pid_inputs[i, :inst_size] = pids
pid_inputs[i, inst_size:] = PAD_ID_TAG
# type ids
tid_inputs[i, :inst_size] = tids
tid_inputs[i, inst_size:] = PAD_ID_TAG
# heads
hid_inputs[i, :inst_size] = hids
hid_inputs[i, inst_size:] = PAD_ID_TAG
# masks
masks[i, :inst_size] = 1.0
for j, wid in enumerate(wids):
if word_alphabet.is_singleton(wid):
single[i, j] = 1
words = torch.from_numpy(wid_inputs)
chars = torch.from_numpy(cid_inputs)
pos = torch.from_numpy(pid_inputs)
heads = torch.from_numpy(hid_inputs)
types = torch.from_numpy(tid_inputs)
masks = torch.from_numpy(masks)
single = torch.from_numpy(single)
lengths = torch.from_numpy(lengths)
data_tensor = {
'WORD': words,
'CHAR': chars,
'POS': pos,
'HEAD': heads,
'TYPE': types,
'MASK': masks,
'SINGLE': single,
'LENGTH': lengths
}
data_tensors.append(data_tensor)
return data_tensors, bucket_sizes
def read_data(source_path,
word_alphabet,
char_alphabet,
pos_alphabet,
type_alphabet,
max_size=None,
normalize_digits=True,
prior_order='inside_out'):
data = []
max_length = 0
max_char_length = 0
print('Reading data from %s' % source_path)
counter = 0
reader = CoNLLXReader(source_path, word_alphabet, char_alphabet,
pos_alphabet, type_alphabet)
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
while inst is not None and (not max_size or counter < max_size):
counter += 1
if counter % 10000 == 0:
print("reading data: %d" % counter)
sent = inst.sentence
stacked_heads, children, siblings, stacked_types, skip_connect = _generate_stack_inputs(
inst.heads, inst.type_ids, prior_order)
data.append([
sent.word_ids, sent.char_id_seqs, inst.pos_ids, inst.heads,
inst.type_ids, stacked_heads, children, siblings, stacked_types,
skip_connect
])
max_len = max([len(char_seq) for char_seq in sent.char_seqs])
if max_char_length < max_len:
max_char_length = max_len
if max_length < inst.length():
max_length = inst.length()
inst = reader.getNext(normalize_digits=normalize_digits,
symbolic_root=True,
symbolic_end=False)
reader.close()
print("Total number of data: %d" % counter)
data_size = len(data)
char_length = min(MAX_CHAR_LENGTH, max_char_length)
wid_inputs = np.empty([data_size, max_length], dtype=np.int64)
cid_inputs = np.empty([data_size, max_length, char_length], dtype=np.int64)
pid_inputs = np.empty([data_size, max_length], dtype=np.int64)
hid_inputs = np.empty([data_size, max_length], dtype=np.int64)
tid_inputs = np.empty([data_size, max_length], dtype=np.int64)
masks_e = np.zeros([data_size, max_length], dtype=np.float32)
single = np.zeros([data_size, max_length], dtype=np.int64)
lengths = np.empty(data_size, dtype=np.int64)
stack_hid_inputs = np.empty([data_size, 2 * max_length - 1],
dtype=np.int64)
chid_inputs = np.empty([data_size, 2 * max_length - 1], dtype=np.int64)
ssid_inputs = np.empty([data_size, 2 * max_length - 1], dtype=np.int64)
stack_tid_inputs = np.empty([data_size, 2 * max_length - 1],
dtype=np.int64)
skip_connect_inputs = np.empty([data_size, 2 * max_length - 1],
dtype=np.int64)
masks_d = np.zeros([data_size, 2 * max_length - 1], dtype=np.float32)
for i, inst in enumerate(data):
wids, cid_seqs, pids, hids, tids, stack_hids, chids, ssids, stack_tids, skip_ids = inst
inst_size = len(wids)
lengths[i] = inst_size
# word ids
wid_inputs[i, :inst_size] = wids
wid_inputs[i, inst_size:] = PAD_ID_WORD
for c, cids in enumerate(cid_seqs):
cid_inputs[i, c, :len(cids)] = cids
cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
# pos ids
pid_inputs[i, :inst_size] = pids
pid_inputs[i, inst_size:] = PAD_ID_TAG
# type ids
tid_inputs[i, :inst_size] = tids
tid_inputs[i, inst_size:] = PAD_ID_TAG
# heads
hid_inputs[i, :inst_size] = hids
hid_inputs[i, inst_size:] = PAD_ID_TAG
# masks_e
masks_e[i, :inst_size] = 1.0
for j, wid in enumerate(wids):
if word_alphabet.is_singleton(wid):
single[i, j] = 1
inst_size_decoder = 2 * inst_size - 1
# stacked heads
stack_hid_inputs[i, :inst_size_decoder] = stack_hids
stack_hid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# children
chid_inputs[i, :inst_size_decoder] = chids
chid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# siblings
ssid_inputs[i, :inst_size_decoder] = ssids
ssid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# stacked types
stack_tid_inputs[i, :inst_size_decoder] = stack_tids
stack_tid_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# skip connects
skip_connect_inputs[i, :inst_size_decoder] = skip_ids
skip_connect_inputs[i, inst_size_decoder:] = PAD_ID_TAG
# masks_d
masks_d[i, :inst_size_decoder] = 1.0
words = torch.from_numpy(wid_inputs)
chars = torch.from_numpy(cid_inputs)
pos = torch.from_numpy(pid_inputs)
heads = torch.from_numpy(hid_inputs)
types = torch.from_numpy(tid_inputs)
masks_e = torch.from_numpy(masks_e)
single = torch.from_numpy(single)
lengths = torch.from_numpy(lengths)
stacked_heads = torch.from_numpy(stack_hid_inputs)
children = torch.from_numpy(chid_inputs)
siblings = torch.from_numpy(ssid_inputs)
stacked_types = torch.from_numpy(stack_tid_inputs)
skip_connect = torch.from_numpy(skip_connect_inputs)
masks_d = torch.from_numpy(masks_d)
data_tensor = {
'WORD': words,
'CHAR': chars,
'POS': pos,
'HEAD': heads,
'TYPE': types,
'MASK_ENC': masks_e,
'SINGLE': single,
'LENGTH': lengths,
'STACK_HEAD': stacked_heads,
'CHILD': children,
'SIBLING': siblings,
'STACK_TYPE': stacked_types,
'SKIP_CONNECT': skip_connect,
'MASK_DEC': masks_d
}
return data_tensor, data_size
def read_data(source_path: str, word_alphabet: Alphabet, char_alphabet: Alphabet, pos_alphabet: Alphabet, type_alphabet: Alphabet,
max_size=None, normalize_digits=True, symbolic_root=False, symbolic_end=False):
data = []
max_length = 0
max_char_length = 0
print('Reading data from %s' % source_path)
counter = 0
data_tensors = []
reader = CoNLLXReader(source_path, word_alphabet, char_alphabet, pos_alphabet, type_alphabet)
inst = reader.getNext(normalize_digits=normalize_digits, symbolic_root=symbolic_root, symbolic_end=symbolic_end)
while inst is not None and (not max_size or counter < max_size):
counter += 1
if counter % 10000 == 0:
print("reading data: %d" % counter)
sent = inst.sentence
data.append([sent.bert_ids, sent.sub_idx, sent.word_ids, sent.char_id_seqs, inst.pos_ids, inst.heads, inst.type_ids])
max_len = max([len(char_seq) for char_seq in sent.char_seqs])
if max_char_length < max_len:
max_char_length = max_len
if max_length < inst.length():
max_length = inst.length()
inst = reader.getNext(normalize_digits=normalize_digits, symbolic_root=symbolic_root, symbolic_end=symbolic_end)
reader.close()
print("Total number of data: %d" % counter)
data_size = len(data)
char_length = min(MAX_CHAR_LENGTH, max_char_length)
bert_wid_inputs = np.empty([data_size, max_length], dtype=np.int64)
subword_word_indicator = np.empty([data_size, max_length], dtype=np.int64)
wid_inputs = np.empty([data_size, max_length], dtype=np.int64)
cid_inputs = np.empty([data_size, max_length, char_length], dtype=np.int64)
pid_inputs = np.empty([data_size, max_length], dtype=np.int64)
hid_inputs = np.empty([data_size, max_length], dtype=np.int64)
tid_inputs = np.empty([data_size, max_length], dtype=np.int64)
masks = np.zeros([data_size, max_length], dtype=np.float32)
single = np.zeros([data_size, max_length], dtype=np.int64)
lengths = np.empty(data_size, dtype=np.int64)
bert_lengths = np.empty(data_size, dtype=np.int64)
for i, inst in enumerate(data):
bert_wids, subword_ids, wids, cid_seqs, pids, hids, tids = inst
inst_size = len(wids)
bpe_st_size = len(bert_wids)
lengths[i] = inst_size
bert_lengths[i] = bpe_st_size
# word ids
wid_inputs[i, :inst_size] = wids
wid_inputs[i, inst_size:] = PAD_ID_WORD
# bert ids
bert_wid_inputs[i, :bpe_st_size] = bert_wids
bert_wid_inputs[i, bpe_st_size:] = 0
# subword
subword_word_indicator[i, :len(subword_ids)] = subword_ids
subword_word_indicator[i, len(subword_ids):] = 0
for c, cids in enumerate(cid_seqs):
cid_inputs[i, c, :len(cids)] = cids
cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
# pos ids
pid_inputs[i, :inst_size] = pids
pid_inputs[i, inst_size:] = PAD_ID_TAG
# type ids
tid_inputs[i, :inst_size] = tids
tid_inputs[i, inst_size:] = PAD_ID_TAG
# heads
hid_inputs[i, :inst_size] = hids
hid_inputs[i, inst_size:] = PAD_ID_TAG
# masks
masks[i, :inst_size] = 1.0
for j, wid in enumerate(wids):
if word_alphabet.is_singleton(wid):
single[i, j] = 1
bert_wid_inputs = torch.from_numpy(bert_wid_inputs)
words = torch.from_numpy(wid_inputs)
chars = torch.from_numpy(cid_inputs)
pos = torch.from_numpy(pid_inputs)
heads = torch.from_numpy(hid_inputs)
types = torch.from_numpy(tid_inputs)
masks = torch.from_numpy(masks)
single = torch.from_numpy(single)
lengths = torch.from_numpy(lengths)
bert_lengths = torch.from_numpy(bert_lengths)
subword_word_indicator = torch.from_numpy(subword_word_indicator)
data_tensor = {'WORD': words, 'CHAR': chars, 'POS': pos, 'HEAD': heads, 'TYPE': types,
'MASK': masks, 'SINGLE': single, 'LENGTH': lengths, 'BERT_LENGTH': bert_lengths,
'BERT_WORD': bert_wid_inputs, 'SUB_IDX': subword_word_indicator}
return data_tensor, data_size