def __init__(self, fnname, params_str, body_str): assert isinstance(params_str, str) self._fnname = fnname self._body_str = body_str if DEBUGGING: for i in range(len(body_str)): if body_str[i] == "`": WarningMsg( "Looks like you've used a backquote in a macro definition. As of now, this probably means your code will only run in a js interpreter that supports template strings (es6)." ) possibly_typed_params_list_chunks = split_by_top_level_commas( Chunk.fromSingleLine(params_str)) possibly_typed_params_list = list( map(lambda x: x.asSingleLine(), possibly_typed_params_list_chunks)) self._numRestParams = 1 if len( possibly_typed_params_list ) > 0 and "..." in possibly_typed_params_list[-1] else 0 self._numOptionalParams = sum( map(lambda x: 1 if ("?" in x) else 0, possibly_typed_params_list)) self._numParams = len(possibly_typed_params_list) self._numRegParams = self._numParams - self._numOptionalParams - self._numRestParams self._params = list( map(MacroDefn._parse_typed_param, possibly_typed_params_list)) self._param2ind = dict() for i in range(len(self._params)): self._param2ind[self._params[i]] = i self._params_re = re.compile("|".join(self._params)) self._param_occurrences = self._findParamOccurrences() if len(self._param_occurrences) == 0: self._no_subst = True self._fillTemplate()
line_info = line.split() idx = int(line_info[1]) dst = int(re.search(r'(.*?)D', line_info[2]).group(1)) if (dst in srcs_dict) and (dst != -1): srcs_dict[dst].append(idx) else: srcs_dict[dst] = [idx] if idx != 0: chunks.append(chunk) else: chunks = [] chunk = Chunk(idx, dst) elif line == "EOS": chunks.append(chunk) for k, v in srcs_dict.items(): chunks[k].update_srsc(v) sents.append(chunks) srcs_dict.clear() else: morph = Morph(line) chunk.update_morph(morph) for sent in sents: for m in sent:
srcs_list = [] sentence_list = [] sentence_list_temp = [] # リセット morph_list = [] next chunk_len = len(dst_list) for i in range(chunk_len): if str(i) in dst_list: srcs_list.append([ str(s) for s, x in enumerate(dst_list) if x == str(i) ]) else: srcs_list.append(["-1"]) sentence = "".join(sentence_list) if len(sentence) > 0: all_sent.append( Chunk(sentence_list, morph_list, dst_list, srcs_list)) one_sent = [] dst_list = [] one_morph = [] one_chank = [] srcs_list = [] sentence_list = [] sentence_list_temp = [] morph_list = [] with open("data/all_sent.pkl", "wb") as f: pickle.dump(all_sent, f)