def get_extended_class(start_line=None):
# Figure out if we're in an inner class and return its extended type if so.
if not start_line:
start_line = util.get_cursor_line_num()
start_indent = util.get_indent(start_line)
if start_indent > 0:
for decl in iter_decls(start_line, -1, FUNC_DECLS | CLASS_DECLS):
indent = util.get_indent(decl.line)
if indent == start_indent:
continue
decl_type = type(decl)
if decl_type is FuncDecl:
start_indent = indent
elif decl_type is ClassDecl:
if decl.extends:
return decl.extends
else:
return None
if indent == 0:
break
# Search for 'extends' at the top of the file.
for lnum in range(1, util.get_line_count()):
line = util.get_line(lnum)
m = re.match("extends\s+(\w+)", line)
if m:
return m.group(1)
# Only 'tool' can appear before 'extends', so stop searching if any other
# text is encountered.
elif not re.match("tool\s+$", line):
return None
def _iter_decls_down(start_line, flags):
# Check whether the starting line is a class decl.
# If so, the indent of the next line is used as a baseline to determine
# which items are direct children of the inner class.
in_class = False
class_decl = _get_decl(start_line, CLASS_DECLS)
if class_decl:
in_class = True
class_indent = util.get_indent(start_line)
inner_indent = None
if flags & CLASS_DECLS:
yield class_decl
for lnum in range(start_line + 1, util.get_line_count()):
if not util.get_line(lnum):
continue
indent = util.get_indent(lnum)
if in_class:
if indent <= class_indent:
return
if not inner_indent:
inner_indent = indent
elif indent > inner_indent:
continue
else:
if indent > 0:
continue
decl = _get_decl(lnum, flags)
if decl:
yield decl
def get_enum_values(line_num):
lines = [util.strip_line(line_num, util.get_line(line_num))]
line_count = util.get_line_count()
while not lines[-1].endswith("}"):
line_num += 1
if line_num > line_count:
return
lines.append(util.strip_line(line_num, util.get_line(line_num)))
m = re.match(_ENUM_VALUES_PATTERN, "\n".join(lines), re.DOTALL)
if m:
values = [v.strip() for v in m.group(1).replace("\n", ",").split(",")]
def map_value(v):
m = re.match("(\w+)(?:\s*=\s*(.*))?", v)
if m:
return ConstDecl(-1, m.group(1), m.group(2))
return list(filter(lambda v: v, map(map_value, values)))
dev_list_context_tag.append(p_token_tag)
dev_list_context_ner.append(p_token_ner)
dev_list_context_char.append(p_token_char)
dev_list_question.append(q_token_text)
dev_list_question_pos.append(q_token_pos)
dev_list_question_tag.append(q_token_tag)
dev_list_question_ner.append(q_token_ner)
dev_list_question_char.append(q_token_char)
dev_spans.append([token_s_id, token_e_id])
if i % 500 == 0:
print("Processed %d of %d (%f percent done)" %
(i, len(dev['questions']),
100 * float(i) / float(len(dev['questions']))))
print('preprocess SQUAD data - char simple phase!')
glove_char_length = get_line_count(setting.glove_char_dir)
char_simple_vocab_w2i = load_glove_file_vocab(setting.glove_char_dir)
char_simple_vocab_w2i["--OOV--"] = len(char_simple_vocab_w2i)
char_simple_vocab_w2i["--PAD--"] = len(char_simple_vocab_w2i)
char_simple_vocab_i2w = {}
for index, word in enumerate(char_simple_vocab_w2i):
char_simple_vocab_i2w[char_simple_vocab_w2i[word]] = word
char_Vocab_simple = Vocab_class(char_simple_vocab_w2i,
char_simple_vocab_i2w)
char_Vocab_simple.save(
os.path.join(SQUAD_dir, setting.char_simple_vocab_w2i_file),
os.path.join(SQUAD_dir, setting.char_simple_vocab_i2w_file))
train_P_char_simple = create_char_array(list_context_char,
char_simple_vocab_w2i,