def 提華語句法樹(bunji="我 喜歡 豬", url='http://localhost:9000'): try: 句法分析器 = CoreNLPParser(url=url) except Warning as 錯誤: print('Warning=', 錯誤) 分析結果指標 = 句法分析器.parse(simplify(bunji).split()) 該句結果字串 = next(分析結果指標) return 該句結果字串 # 印字串 # (ROOT (IP (NP (PN 我)) (VP (VV 喜欢) (NP (NN 猪))))) print('該句結果字串=', 該句結果字串) # 照字串印樹仔圖 # ROOT # | # IP # ___|____ # | VP # | ____|___ # NP | NP # | | | # PN VV NN # | | | # 我 喜欢 猪 該句結果字串.pretty_print() ##### 樹仔字串提出原始字串 a = Tree.fromstring("(ROOT (IP (NP (PN 我)) (VP (VV 喜欢) (NP (NN 猪)))))") # ['我', '喜欢', '猪'] print(a.leaves()) # (ROOT 我 喜欢 猪) print(a.flatten())
def Load(filename, stopwords, output_filename): global MIN_PERCENT global MIN_SUP candidate = set() for line in codecs.open(filename, 'r', 'utf-8'): if line[0].isdigit(): continue tokens = line.strip().split('\t') valid = False for token in tokens[2:]: support = int(token.split(':')[-2]) percentage = float(token.split(':')[-1][:-1]) if (percentage >= MIN_PERCENT) or (support >= MIN_SUP): name = ':'.join(token.split(':')[:-2]) valid = True if NoSeparator(name) and StopWordChecking(name, stopwords): candidate.add(name.lower()) if valid: name = tokens[0] if NoSeparator(name) and StopWordChecking(name, stopwords): candidate.add(name.lower()) out = codecs.open(output_filename, 'w', 'utf-8') if LANGUAGE == 'zh': seen = set() for name in candidate: name = simplify(''.join(name.split())) seen.add(name) candidate = seen print len(candidate) for name in candidate: out.write(name + '\n') out.close()
def seg(sentence): paraphrase = tokenizer.encode_plus(simplify(sentence), return_tensors="pt") paraphrase['attention_mask'][-1] = 0 # pdb.set_trace() for key in paraphrase.keys(): paraphrase[key] = paraphrase[key].to(device) paraphrase_classification_logits = model(**paraphrase)[0] paraphrase_results = paraphrase_classification_logits.argmax(axis=-1)[0] paraphrase_results = paraphrase_results[1:-1] # pdb.set_trace() res = list() length = 0 word = False for i in range(len(paraphrase_results)): if paraphrase_results[i] == 3: res.append(sentence[i]) length = 0 if paraphrase_results[i] == 0: length += 1 if paraphrase_results[i] == 1: length += 1 if paraphrase_results[i] == 2: res.append(sentence[i-length:i+1]) length = 0 #print(''.join(res) == sentence) print(' '.join(res), '\n') return ','.join(res) + '\n'
def Load(filename, stopwords, output_filename): global MIN_PERCENT global MIN_SUP candidate = set() for line in codecs.open(filename, 'r', 'utf-8'): if line[0].isdigit(): continue tokens = line.strip().split('\t') valid = False for token in tokens[3:]: support = int(token.split(':')[-2]) percentage = float(token.split(':')[-1][:-1]) if (percentage >= MIN_PERCENT) or (support >= MIN_SUP): name = ':'.join(token.split(':')[:-2]) valid = True if NoSeparator(name) and StopWordChecking(name, stopwords): candidate.add(name.lower()) if valid: name = tokens[0] if NoSeparator(name) and StopWordChecking(name, stopwords): candidate.add(name.lower()) out = codecs.open(output_filename, 'w', 'utf-8') if LANGUAGE == 'zh': seen = set() for name in candidate: name = simplify(''.join(name.split())) seen.add(name) candidate = seen print len(candidate) for name in candidate: out.write(name + '\n') out.close()
def preprocess_text(text, truncate_at=100): ''' 清除限定字符集外的内容,并截取前若干字符的文本. ''' truncated = text[:truncate_at] cleaned = REGEX_TO_REMOVE.sub(r'', truncated) return mafan.simplify(cleaned)
def can_st(page): simplified = simplify(page) == page.decode("utf8") traditional = tradify(page) == page.decode("utf8") # only simplified if simplified and not traditional and not config.zh_s: return False # only traditional elif traditional and not simplified and not config.zh_t: return False else: return config.zh_t or config.zh_s
def can_st(page): simplified = simplify(page) == page.decode("utf8") traditional = tradify(page) == page.decode("utf8") # only simplified if simplified and not traditional and not config.zh_s: return False # only traditional elif traditional and not simplified and not config.zh_t: return False else: return config.zh_t or config.zh_s
def Load(filename, output_filename): candidate = set() for line in codecs.open(filename, 'r', 'utf-8'): tokens = line.strip().split('\t') for token in tokens[3:]: name = ':'.join(token.split(':')[:-2]) if LANGUAGE == 'zh': name = simplify(''.join(name.split())) candidate.add(name.lower()) name = tokens[0] #name = simplify(''.join(name.split(''))) if LANGUAGE == 'zh': name = simplify(''.join(name.split())) candidate.add(name.lower()) print len(candidate) out = codecs.open(output_filename, 'w', 'utf-8') for name in candidate: out.write(name + '\n') out.close()
def TM(filename, k, weights, simpmode=False): sys.stderr.write("Reading translation model from %s...\n" % (filename, )) tm = {} for line in open(filename).readlines(): (f, e, features) = line.strip().split(" ||| ") tm.setdefault(tuple(f.split()), []).append( phrase(e, [float(i) for i in features.strip().split()])) tmptm = {} for f in tm: # prune all but top k translations tm[f].sort(key=lambda x: sum(p * q for p, q in zip(x.features, weights)), reverse=True) del tm[f][k:] if simpmode: from mafan import simplify sf = tuple(simplify(f[i].decode('utf-8')) for i in range(len(f))) if sf != tuple(f[i].decode('utf-8') for i in range(len(f))): if sf in tm: for p in tm[f]: found = False for pi, sp in enumerate(tm[sf]): if sp.english == p.english: found = True tm[sf][pi].features[0] = ( tm[sf][pi].features[0] + p.features[0]) / 2 tm[sf][pi].features[1] = math.log( math.exp(tm[sf][pi].features[1]) + math.exp(p.features[1])) tm[sf][pi].features[2] = max( [tm[sf][pi].features[2], p.features[2]]) tm[sf][pi].features[3] = max( [tm[sf][pi].features[3], p.features[3]]) if not found: tm[sf].append( phrase(p.english, [ p.features[0] / 2, math.log(math.exp(p.features[1]) + 1), p.features[2], p.features[3] ])) else: for p in tm[f]: tmptm.setdefault(sf, []).append(p) for f in tmptm: tm[f] = tmptm[f] return tm
def StdNm(nonstd=None): ## make a DataFrame indexed by standardized name and ## contains columns "FirstName", "OtherNames", "OtherNames1" std = pd.read_csv("csv/StandardNames.csv") std["FullName"] = std["LastName"] + std["FirstName"] std["FullName"].fillna(method="ffill", inplace=True) std.set_index("FullName", inplace=True) #### Simplified characters for LastName happen sometimes std["ConvLast"] = [ sTR if pd.isnull(sTR) else mf.simplify(sTR) if mf.is_traditional(sTR) else mf.tradify(sTR) for sTR in std["LastName"] ] std["LastOth"] = std["LastName"].fillna(method="ffill") + std['OtherNames'] std["ConvFst"] = std["ConvLast"].fillna(method="ffill") + std['FirstName'] std["ConvOth"] = std["ConvLast"].fillna(method="ffill") + std['OtherNames'] std.drop(["Details", "Studio", "LastName", "ConvLast"], axis=1, inplace=True) ## make a dataframe of {key: alternative names, value: standard names} with ## unique keys and overlapping values map_df = pd.DataFrame() for colName in list(std.columns): df = pd.DataFrame({"key": std[colName], "value": std.index}) map_df = map_df.append(df, ignore_index=True) map_df.dropna(inplace=True) ## Standardize names in the given Series map_dict = map_df.set_index('key').to_dict()['value'] def standardize_names(participant): if participant in map_dict: return map_dict[participant] else: return participant ans = nonstd.map(standardize_names) return ans
def extract_glove_embeddings(): log("extract_glove_embeddings()...") _, word_to_index = read_list_file(word_file) word_list = [] embedding_list = [] with codecs.open(glove_file, "r", encoding="utf8") as f: for line in f: line = line.strip().split() word = mafan.simplify(line[0]) if word not in word_to_index: continue embedding = np.array([float(i) for i in line[1:]]) word_list.append(word) embedding_list.append(embedding) np.save(pretrained_word_file, word_list) np.save(pretrained_embedding_file, embedding_list) log(" %d pre-trained words\n" % len(word_list)) return
def extract_glove_embeddings(): log("extract_glove_embeddings()...") _, word_to_index = read_list_file(word_file) word_list = [] embedding_list = [] with codecs.open(glove_file, "r", encoding="utf8") as f: for line in f: line = line.strip().split() word = mafan.simplify(line[0]) if word not in word_to_index: continue embedding = np.array([float(i) for i in line[1:]]) word_list.append(word) embedding_list.append(embedding) np.save(pretrained_word_file, word_list) np.save(pretrained_embedding_file, embedding_list) log(" %d pre-trained words\n" % len(word_list)) return
def process_language(ele): try: text_list = ele.text.split('\n') text_list = [replace_special(x) for x in text_list] languages = [guess_language(l) for l in text_list] en_index = [l == 'en' for l in languages] zh_index = [l == 'zh' for l in languages] en_lines = list(compress(text_list, en_index)) zh_lines = list(compress(text_list, zh_index)) if len(en_lines) == 0 or len(zh_lines) == 0: return None en_text = ' '.join(en_lines) if len(check_chinese(en_text)) > 0: return None zh_text = simplify(' '.join(zh_lines)) if len(check_english(zh_text)) > 0: return None text_line = en_text + ' | ' + zh_text return text_line except: return None
def dup_check(self, subtitles): ''' Check if there are dplicated subtitles. :returns: subtitles Subtitles without duplication. ''' # convert all encoding to utf-8, trad -> simp for sub in subtitles: enc = chardet.detect(sub['content'])['encoding'] # for simp. chinese if enc == 'GB2312': sub['content'] = sub['content'].decode('gb18030') else: sub['content'] = sub['content'].decode(enc) # for big5, convert! if enc == 'Big5': sub['content'] = mafan.simplify(sub['content']) # find dups dup_tags = [False] * len(subtitles) for i in range(len(subtitles)): for j in range(i+1, len(subtitles)): sub_a = subtitles[i]['content'] sub_b = subtitles[j]['content'] sim = difflib.SequenceMatcher(None, sub_a, sub_b).real_quick_ratio() logging.debug('Similar ratio between %d and %d is %f' % (i, j, sim)) if sim >= 0.9: if len(sub_a) >= len(sub_b): dup_tags[j] = True else: dup_tags[i] = True new_subs = [s for i, s in enumerate(subtitles) if dup_tags[i] == False] logging.debug('Total %d subtitles remains.' % len(new_subs)) return new_subs
def get_candidates(inputfile, tm, lm, weights, stack_size=10, nbest=None, simpmode=True, separate_unknown_words=False, verbose=False): if nbest is None: nbest = stack_size print >> sys.stderr, "Decoding: " + inputfile print >> sys.stderr, "Reading input..." french = [line.strip().split() for line in open(inputfile).readlines()] # list of list if simpmode: from mafan import simplify for li, line in enumerate(french): for wi, word in enumerate(line): french[li][wi] = simplify(word.decode('utf-8')).encode('utf-8') # tm should translate unknown words as-is with a small probability # (i.e. only fallback to copying unknown words over as the last resort) for i in xrange(len(french)): j = 0 while j < len(french[i]): word = french[i][j] if (word, ) not in tm: flag = True if len(word) >= 2 and separate_unknown_words: for separate in xrange(1, len(word)): if (word[:separate], ) in tm and ( word[separate:], ) in tm: french[i][j] = word[:separate] j += 1 french[i].insert(j, word[separate:]) flag = False break if flag: tm[(word, )] = [ models.phrase(word, [unknown_word_logprob] * number_of_features_PT) ] j += 1 print >> sys.stderr, "Start decoding..." for n, f in enumerate(french): if verbose: print >> sys.stderr, "Input: " + ' '.join(f) # Generate cache for phrase segmentations. f_cache = generate_phrase_cache(f, tm) # Pre-calculate future cost table future_cost_table = precalcuate_future_cost( f, tm, weights[:number_of_features_PT]) # score = dot(features, weights) # features = sums of each log feature # predecessor = previous hypothesis # lm_state = N-gram state (the last one or two words) # last_frange = (i, j) the range of last translated phrase in f # phrase = the last TM phrase object (correspondence to f[last_frange]) # coverage = bit string representing the translation coverage on f # future_cost = a safe estimation to be added to total_score hypothesis = namedtuple( "hypothesis", "score, features, lm_state, predecessor, last_frange, phrase, coverage, future_cost" ) initial_hypothesis = hypothesis(0.0, [0.0] * number_of_features, lm.begin(), None, (0, 0), None, 0, 0) # stacks[# of covered words in f] (from 0 to |f|) stacks = [{} for _ in xrange(len(f) + 1)] # stacks[size][(lm_state, last_frange[1], coverage)]: # recombination based on (lm_state, last_frange[1], coverage). # For different hypotheses with the same tuple, keep the one with the higher score. # lm_state affects LM; last_frange affects distortion; coverage affects available choices. stacks[0][(lm.begin(), None, 0)] = initial_hypothesis for i, stack in enumerate(stacks[:-1]): if verbose: print >> sys.stderr, "Stack[%d]:" % i # Top-k pruning s_hypotheses = sorted(stack.values(), key=lambda h: h.score + h.future_cost, reverse=True) for h in s_hypotheses[:stack_size]: if verbose: print >> sys.stderr, h.score, h.lm_state, bin( h.coverage), ' '.join(f[h.last_frange[0]:h. last_frange[1]]), h.future_cost for (f_range, delta_coverage, tm_phrases) in enumerate_phrases(f_cache, h.coverage): # f_range = (i, j) of the enumerated next phrase to be translated # delta_coverage = coverage of f_range # tm_phrases = TM entries corresponding to fphrase f[f_range] length = i + f_range[1] - f_range[0] coverage = h.coverage | delta_coverage distance = abs(f_range[0] - h.last_frange[1]) # if distance > max_distance and i < len(stacks) / 2: # continue # TM might give us multiple candidates for a fphrase. for phrase in tm_phrases: features = h.features[:] # copy! # Features from phrase table for fid in range(number_of_features_PT): features[fid] += phrase.features[fid] # log_lmprob (N-gram) lm_state = h.lm_state loglm = 0.0 for word in phrase.english.split(): (lm_state, word_logprob) = lm.score(lm_state, word) loglm += word_logprob # Don't forget the STOP N-gram if we just covered the whole sentence. loglm += lm.end(lm_state) if length == len(f) else 0.0 features[4] += loglm # log distortion (distance ** alpha) features[5] += log(alpha) * distance # length of the translation (-length) features[6] += -len(phrase.english.split()) score = calculate_total_score(features, weights) future_list = get_future_list(coverage, len(f)) future_cost = get_future_cost(future_list, future_cost_table) new_state = (lm_state, f_range[1], coverage) new_hypothesis = hypothesis(score, features, lm_state, h, f_range, phrase, coverage, future_cost) # Recombination if new_state not in stacks[length] or \ score + future_cost > stacks[length][new_state].score + stacks[length][new_state].future_cost: stacks[length][new_state] = new_hypothesis winners = sorted(stacks[len(f)].values(), key=lambda h: h.score, reverse=True) if nbest == 1: yield extract_english(winners[0]) else: for s in winners[:nbest]: yield ("%d ||| %s |||" + " %f" * number_of_features) % \ ((n, extract_english(s)) + tuple(s.features)) print >> sys.stderr, "Decoding completed"
def get_candidates(input, tm, lm, weights, s=1): alpha = 0.95 #reordering parameter french = [list(line.strip().split()) for line in open(input).readlines()] for li, line in enumerate(french): for wi, word in enumerate(line): french[li][wi] = simplify(word.decode('utf-8')).encode('utf-8') # tm should translate unknown words as-is with probability 1 for word in set(sum(french, [])): if (word, ) not in tm: tm[(word, )] = [models.phrase(word, [0.0, 0.0, 0.0, 0.0])] def generate_phrase_cache(f): cache = [] for i in range(0, len(f)): entries = [] bitstring = 0 for j in range(i + 1, len(f) + 1): bitstring += 1 << (len(f) - j) if tuple(f[i:j]) in tm: entries.append({ 'end': j, 'bitstring': bitstring, 'phrase': tm[tuple(f[i:j])] }) cache.append(entries) return cache def enumerate_phrases(f_cache, coverage): for i in range(0, len(f_cache)): bitstring = 0 for entry in f_cache[i]: if (entry['bitstring'] & coverage) == 0: yield ((i, entry['end']), entry['bitstring'], entry['phrase']) def precalcuate_future_cost(f): phraseCheapestTable = {} futureCostTable = {} for i in range(0, len(f)): for j in range(i + 1, len(f) + 1): if f[i:j] in tm: phraseCheapestTable[i, j] = -sys.maxint for phrase in tm[f[i:j]]: if phrase.logprob > phraseCheapestTable[i, j]: phraseCheapestTable[i, j] = phrase.logprob for i in range(0, len(f)): futureCostTable[i, 1] = phraseCheapestTable[i, i + 1] for j in range(2, len(f) + 1 - i): if (i, i + j) in phraseCheapestTable: futureCostTable[i, j] = phraseCheapestTable[i, i + j] else: futureCostTable[i, j] = -sys.maxint for k in range(1, j): if (((i + k, i + j) in phraseCheapestTable) and (futureCostTable[i, j] < futureCostTable[i, k] + phraseCheapestTable[i + k, i + j])): futureCostTable[i, j] = futureCostTable[ i, k] + phraseCheapestTable[i + k, i + j] return futureCostTable def get_future_list(bitstring): bitList = bin(bitstring)[2:] futureList = [] count = 0 index = 0 findZeroBit = False for i in range(len(bitList)): if bitList[i] == '0': if not findZeroBit: index = i findZeroBit = True count = count + 1 else: if findZeroBit: futureList.append((index, count)) findZeroBit = False count = 0 if findZeroBit: futureList.append((index, count)) return futureList def get_future_cost(bitList, futureCostTable): cost = 0 for item in bitList: cost = cost + futureCostTable[item] return cost def extract_english(h): return "" if h.predecessor is None else "%s%s " % (extract_english( h.predecessor), h.phrase.english) results = [] sys.stderr.write("Decoding %s...\n" % (input, )) for n, f in enumerate(french): # Generate cache for phrase segmentations. f_cache = generate_phrase_cache(f) # Pre-calculate future cost table #future_cost_table = precalcuate_future_cost(f) # logprob = log_lmprob + log_tmprob + distortion_penalty # predecessor = previous hypothesis # lm_state = N-gram state (the last one or two words) # last_frange = (i, j) the range of last translated phrase in f # phrase = the last TM phrase object (correspondence to f[last_frange]) # coverage = bit string representing the translation coverage on f # future_cost hypothesis = namedtuple( "hypothesis", "logprob, features, lm_score, lm_state, predecessor, last_frange, phrase, coverage" ) initial_hypothesis = hypothesis(0.0, [0.0, 0.0, 0.0, 0.0], 0.0, lm.begin(), None, (0, 0), None, 0) # stacks[# of covered words in f] (from 0 to |f|) stacks = [{} for _ in range(len(f) + 1)] # stacks[size][(lm_state, last_frange, coverage)]: # recombination based on (lm_state, last_frange, coverage). # For different hypotheses with the same tuple, keep the one with the higher logprob. # lm_state affects LM; last_frange affects distortion; coverage affects available choices. stacks[0][(lm.begin(), None, 0)] = initial_hypothesis for i, stack in enumerate(stacks[:-1]): # Top-k pruning for h in sorted(stack.itervalues(), key=lambda h: -h.logprob)[:s]: for (f_range, delta_coverage, tm_phrases) in enumerate_phrases(f_cache, h.coverage): # f_range = (i, j) of the enumerated next phrase to be translated # delta_coverage = coverage of f_range # tm_phrases = TM entries corresponding to fphrase f[f_range] length = i + f_range[1] - f_range[0] coverage = h.coverage | delta_coverage distance = f_range[0] - h.last_frange[1] # TM might give us multiple candidates for a fphrase. for phrase in tm_phrases: # log_tmprob and distortion features = map(add, h.features, phrase.features) # log_lmprob (N-gram) lm_state = h.lm_state lm_score = h.lm_score for word in phrase.english.split(): (lm_state, word_logprob) = lm.score(lm_state, word) lm_score += word_logprob # Don't forget the STOP N-gram if we just covered the whole sentence. lm_score += lm.end(lm_state) if length == len( f) else 0.0 # Future cost. #future_list = get_future_list(delta_coverage) #future_cost = get_future_cost(future_list, future_cost_table) logprob = sum( p * q for p, q in zip((features + [lm_score]), weights)) new_state = (lm_state, f_range, coverage) new_hypothesis = hypothesis(logprob, features, lm_score, lm_state, h, f_range, phrase, coverage) if new_state not in stacks[length] or \ logprob > stacks[length][new_state].logprob: # recombination stacks[length][new_state] = new_hypothesis winner = sorted(stacks[len(f)].itervalues(), key=lambda h: h.logprob, reverse=True)[0:100] for i in range(len(winner)): results += [ "%d ||| %s ||| %f %f %f %f %f" % (n, extract_english(winner[i]), winner[i].features[0], winner[i].features[1], winner[i].features[2], winner[i].features[3], winner[i].lm_score) ] return results
from mafan import encoding from mafan import text from mafan import simplify, tradify from mafan import split_text import mafan from mafan import pinyin # 对包含其他编码的文本转化成utf8格式 # filename = 'test.txt' # name or path of file as string # encoding.convert(filename) # creates a file with name 'ugly_big5_utf-8.txt' in glorious utf-8 encoding # 简体和繁体转化 print('-' * 50) string = u'这是麻烦啦' print(tradify(string)) # convert string to traditional print(simplify(tradify(string))) # convert back to simplified # 是否包含符号或者拉丁字符 print('-' * 50) flag = text.has_punctuation(u'这是麻烦啦') print(flag) flag = text.has_punctuation(u'这是麻烦啦.') print(flag) flag = text.has_punctuation(u'这是麻烦啦。') print(flag) flag = text.contains_latin(u'这是麻烦啦。') print(flag) flag = text.contains_latin(u'You are麻烦啦。') print(flag) # 判断是否是简体还是繁体
def join_elements(candidate): seen = set() for name in candidate: name = simplify(''.join(name.split())) seen.add(name) return seen
def simplify(): text = request.args.get('text') d = {'text': mafan.simplify(text)} return jsonify(**d)
line = fp.readline() while line: content = json.loads(line) text_a = content.get("question") text_b = content.get("answer") label = content.get("yesno_answer") text_a = text_a.replace(' ', '') text_b = text_b.replace(' ', '') text_a = text_a.replace('\t', '') text_b = text_b.replace('\t', '') text_a = text_a.replace('?', '?') text_b = text_b.replace('?', '?') if text_a[-1] != "?": text_a = text_a + "?" text_a = mafan.simplify(text_a) text_b = mafan.simplify(text_b) if label == "" and filename != "test": print("miss label") label = "Yes" '''if filename != "test": out_line = text_a + '\t' + text_b + '\t' + label else: out_line = text_a + '\t' + text_b''' out_line = [text_a, text_b] out_arr.append(out_line) length_arr.append(len(text_a + text_b)) line = fp.readline() print(filename + " aver length is:%d", sum(length_arr) / len(length_arr)) print(filename + " max length is:%d", max(length_arr))
def convert(): return simplify(request.args.get("word"))