def sameCheck(input_string1, input_string2): # 유니코드 한글을 비교하여 유사도 리턴. # 0 - 초,중,종 모두 다름 # 1 - 한개가 같음 # 2 - 두개가 같음 # 3 - 동일한 글자 # 단일 자음은 종성으로 간주하고 convert_dictionary 를 이용하여 유사도 비교 convert_dictionary = { 1: 5, 4: 8, 7: 11, 8: 13, 16: 21, 17: 22, 19: 25, 21: 27, 22: 0, 23: 2, 24: 3, 25: 4, 26: 5, 27: 6, 0: 0 } var1 = hangul.separate(input_string1) var2 = hangul.separate(input_string2) same_point = 0 if var1[0] == -75: # 한글이 아닌 기호여서, full match 만 해야 함 for index in range(0, 3): if var1[index] == var2[index]: if index == 2: if var1[index] != 0: same_point += 1 else: same_point += 1 if same_point == 3: return 3 else: return 0 else: if var2[0] == -54 and var2[1] == 11: # 단독자음 처리 if convert_dictionary[var1[2]] == var2[2]: return 1 for index in range(0, 3): # 한글 처리 if var1[index] == var2[index]: if index == 2: if var1[index] != 0: same_point += 1 else: same_point += 1 return same_point
def get_onehot_vector(sent): """ convert sentecne to vector :return: list """ try: return_vector = [] embeddings = np.zeros([30]) idx = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', ' '] num_reg = re.compile("[0-9- ]") if (type(sent) not in [type('str'), type([])]): raise Exception("input must be str") if (type(sent) == type([])): sent = sent[0] for char in sent: vector_a = np.copy(embeddings) vector_b = np.copy(embeddings) vector_c = np.copy(embeddings) vector_d = np.copy(embeddings) if (num_reg.match(char) == None and hangul.is_hangul(char)): anl = hangul.separate(char) vector_a[anl[0] if anl[0] > 0 else 0] = 1 vector_b[anl[1] if anl[1] > 0 else 0] = 1 vector_c[anl[2] if anl[2] > 0 else 0] = 1 elif (num_reg.match(char)): vector_d[idx.index(char)] = 1 return_vector.append( np.append(vector_a, [vector_b, vector_c, vector_d])) return np.array(return_vector) except Exception as e: print("error on get_onehot_vector : {0}".format(e))
def text2label(string): digit_dict = { '0': '영', '1': '일', '2': '이', '3': '삼', '4': '사', '5': '오', '6': '육', '7': '칠', '8': '팔', '9': '구' } char_length = 72 labels = list() for seq in string: label = list() seq = filter.sub('', seq) for i, c in enumerate(seq): if c == ' ': label.append(70) else: if c.isdigit(): c = digit_dict[c] cho, joong, jong = hangul.separate(c) label.append(one_hot(cho, char_length)) label.append(one_hot(joong + 21, char_length)) if jong: label.append(one_hot(jong + 42, char_length)) labels.append(label) return np.array(labels)
def String_to_Token_List(string): #English or special char.... if re.search(r'[가-힣\d\s\.,\?!]+', string) is None or re.search( r'[가-힣\d\s\.,\?!]+', string).group() != string: print(string) return False regex_DtoS = r'(?:^|[^\d])(\d{4})(?:$|[^\d])' string = re.sub( regex_DtoS, lambda x: re.sub(r'\d{4}', lambda y: Number_to_String(y.group()), x.group()), string) regex_CtoS1 = r"([+-]?\d[\d,]*)[\.]?\d*" regex_CtoS2 = r"(시|명|가지|살|마리|포기|송이|수|톨|통|점|개|벌|척|채|다발|그루|자루|줄|켤레|그릇|잔|마디|상자|사람|곡|병|판)" string = re.sub( regex_CtoS1 + regex_CtoS2, lambda x: re.sub( regex_CtoS1, lambda y: Count_Number(int(y.group())), x.group()), string) regex_NtoS = r"([+-]?\d[\d,]*)[\.]?\d*" string = re.sub(regex_NtoS, lambda x: Read_Number(int(x.group())), string) token_List = [] token_List.append(0) #<EOS> for char in string: if char == " ": token_List.append(2) continue elif char == ".": token_List.append(71) continue elif char == ",": token_List.append(72) continue elif char == "?": token_List.append(73) continue elif char == "!": token_List.append(74) continue elif hangul.is_hangul(char): onset, nucleus, coda = hangul.separate(char) onset += 3 nucleus += 3 + 19 coda += 3 + 19 + 21 token_List.extend([onset, nucleus, coda]) else: raise Exception("Not handled letter") token_List.append(1) #<EOE> return token_List
def convert_hangul_to_index(string):#, size): #string = unicode(string) list = []#np.ndarray([size, 3]) for i in range(len(string)): #exception if not hangul.is_hangul(string[i]): continue char3 = hangul.separate(string[i]) idx = char3[0] + char3[1] * FirNum + char3[2] * FirNum * SecNum list.append([idx]) if len(list)==0: list.append([ClassNum - 1]) return np.array(list)
def get_onehot_vector(self, sent): """ convert sentecne to vector :return: list """ try: return_vector = [] embeddings = np.zeros([40]) idx = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', ' ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] num_reg = re.compile("[a-z0-9- ]") if (type(sent) not in [type('str'), type([])]): raise Exception("input must be str") if (type(sent) == type([])): sent = sent[0] for char in sent: vector_a = np.copy(embeddings) vector_b = np.copy(embeddings) vector_c = np.copy(embeddings) vector_d = np.copy(embeddings) if (num_reg.match(char) == None and hangul.is_hangul(char)): anl = hangul.separate(char) vector_a[anl[0] if anl[0] > 0 else 0] = 1 vector_b[anl[1] if anl[1] > 0 else 0] = 1 vector_c[anl[2] if anl[2] > 0 else 0] = 1 elif (num_reg.match(char)): vector_d[idx.index(char)] = 1 else : vector_d[39] = 1 return_vector.append(np.append(vector_a, [vector_b, vector_c, vector_d])) return np.array(return_vector) except Exception as e: print("error on get_onehot_vector : {0}".format(e))
def komoran_processing(komoran, complex_verb_set): tokenList = list() formerTokenTag = "" original_token = "" word_cnt = len(komoran) word_idx = 0 for phrase in komoran: phrase_cnt = len(phrase) idx = 0 word_idx += 1 tokenWord = "" formerTag = "" new_check = False while idx < phrase_cnt: token = phrase[idx].getFirst().replace(" ", "") #remove white space in proper nouns if " " not in token and len(token) > 9: idx += 1 new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue tag = phrase[idx].getSecond() ########## FIRST PHONEME if idx == 0 or (idx == 1 and formerTag == "XPN"): #XPN: 체언접두사 if formerTokenTag == "REMOVE_EC": tokenList[-1] = "%s " % tokenList[-1] # To combine the former token and the current token elif formerTokenTag == "MAG": tokenWord = tokenList[-1] del tokenList[-1] elif formerTokenTag == "EC": if tag not in {"VV", "VA", "VX"} or (tag in {"VV", "VA", "VX"} and token not in complex_verb_set): tokenList[-1] = original_token formerTokenTag = "" original_token = "" # prefix(체언접두사) if tag == "XPN": tokenWord = token formerTag = tag idx += 1 new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue else: # common noun, proper noun, root adverb and foreign language if tag in {"NNG", "NNP", "SL", "XR", "MAG"}: if tag == "SL": token = token.lower() if tag == "MAG": if token in {"안", "못", "잘못"}: formerTokenTag = "MAG" new_check = True else: idx += 1 new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue elif tag == "NNP": token = "%s " % token elif formerTokenTag == "NNB" and token == "밖": token = "밖에" tokenWord = tokenWord.rstrip("_") formerTokenTag = "JX" new_check = True else: pass # dependent nouns elif tag == "NNB" and token in {"수", "지", "때문"}: if token == "수" and formerTokenTag in {"VV", "VA", "VX"}: formerTokenTag = "NNB1" elif token == "수" and formerTokenTag not in {"VV", "VA", "VX"}: formerTokenTag = "" pass # 사람의 '수' 등의 형태소분석 오류 elif token == "지" and formerTokenTag in {"VV", "VA", "VX"}: formerTokenTag = "NNB2" elif token == "때문": formerTokenTag = "NNB3" else: formerTokenTag = "" new_check = False idx += 1 continue new_check = True try: if formerTokenTag.startswith("NNB"): token = "%s_%s" % (tokenList[-1], token) del tokenList[-1] else: pass except IndexError: # not dependent nouns formerTokenTag = "" new_check = False idx += 1 continue # verb, adjective and auxiliary predicate elif tag in ["VV", "VA", "VX"]: if formerTokenTag == "EC": token = "%s%s" % (tokenList[-1], "%s다" % token) del tokenList[-1] elif formerTokenTag == "JX": if token == "없": # 수밖에_없다 token = "_%s다" % token else: token = " %s다" % token elif formerTokenTag.startswith("NNB"): if formerTokenTag.endswith("1") and token in {"있", "없"}: # 하다_수_있다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] elif formerTokenTag.endswith("2") and token in {"모르", "않", "말"}: # 하다_지_모르다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] elif formerTokenTag.endswith("3"): # 사랑_때문_자라다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] else: token = "%s다" % token # only in case negative words elif token in ["않", "없", "못하", "말", "싫", "주"]: try: token = "%s%s" % (tokenList[-1], "_%s다" % token) del tokenList[-1] except IndexError: token = "_%s다" % token else: token = "%s다" % token formerTokenTag = tag new_check = True elif formerTokenTag == "NNB1" and tag == "JX": if token == "밖에": # 수 밖에 = 수밖에 token = "%s%s" % (tokenList[-1].rstrip("_"), token) del tokenList[-1] formerTokenTag = "JX" new_check = True elif token == "도": # 갈 수도 있다 = 가다_수_있다 new_check = True else: # just pass the rest idx += 1 formerTag = tag new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue tokenWord += token formerTag = tag # checking 'formerTokenTag' to be newly assigned new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) ########## FROM SECOND PHONEME else: if formerTokenTag == "REMOVE_EC": tokenWord = "%s " % tokenWord elif formerTokenTag.startswith("NNB") and tag not in {"VV", "VA", "VX"}: tokenWord = "%s " % tokenWord if formerTag in ["ETM", "ETN"]: # ETM: 관성형전성어미, ETN: 명사형전성어미 tokenWord = "%s " % tokenWord # common noun, proper noun, root and foreign language, suffix(noun) and dependent noun if tag in {"NNG", "NNP", "XR", "SL", "NNB", "XSN"}: if tag == "SL": token = token.lower() if tag == "NNP": if formerTag == "NNP": tokenWord = "%s%s " % (tokenWord, token) else: tokenWord = "%s %s " % (tokenWord, token) elif tag == "NNB": if token in {"수", "지", "때문"}: if token == "수": if formerTag in {"VV", "VA", "VX"}: tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB1" else: tokenWord = "%s %s" % (tokenWord.strip(), token) elif token == "지" and formerTag in {"VV", "VA", "VX"}: tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB2" elif token == "때문": tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB3" else: pass new_check = True else: pass else: tokenWord += token # 부정지정사 '아니' elif tag == "VCN": tokenWord = "%s_%s다" % (tokenWord, token) # 수 밖에 = 수밖에 elif (formerTokenTag == "NNB1" or formerTag == "NNB1") and tag == "JX": if token == "밖에": tokenWord = "%s%s" % (tokenWord.rstrip("_"), token) formerTokenTag = "JX" new_check = True elif token == "도": new_check = True # suffix(adjective and verb) elif tag in {"XSA", "XSV"}: token = "%s다" % token tokenWord += token # connective endings elif tag == "EC": original_token = tokenWord newToken = tokenWord.rstrip("다") try: jong = hangul.separate(newToken[-1]) #마지막 글자 분해 except IndexError: idx += 1 formerTag = tag new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue if token in {"아", "어"}: if formerTag == "VCP": idx += 1 formerTag = tag new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue # 받침없음 if jong[-1] == 0: if jong[1] == 20 and token == "어": # 달리+어=달려 newJong = hangul.build(jong[0], 6, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 18 and jong[0] == 5: # '르' 불규칙: 마르+아=말라 tmp = hangul.separate(newToken[-2]) newJong = hangul.build(tmp[0], tmp[1], 8) tmp2 = hangul.separate(token) newJong2 = hangul.build(5, tmp2[1], tmp2[2]) if len(newToken) == 2: # 마르, 오르, 바르.. newToken = "%s%s" % (newJong, newJong2) else: newToken = "%s%s%s" % (newToken[:-2], newJong, newJong2) tokenWord = newToken elif jong[1] == 18 and token == "어": # 'ㅡ' 불규칙: 쓰+어=써 newJong = hangul.build(jong[0], 4, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 13 and token == "어": # 세우+어=세워 newJong = hangul.build(jong[0], 14, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 8 and token == "아": # 따라오+아=따라와 newJong = hangul.build(jong[0], 9, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 11 and token == "어": # 되+어=되어 newToken = "%s%s" % (newToken, "어") tokenWord = newToken elif jong == (18,0,0) and token in {"아", "어"}: if len(newToken) == 1: tokenWord = "해" else: tokenWord = "%s해" % newToken[:-1] else: tokenWord = newToken # 펴, 자 # 받침있음 else: # '빨개야', '파래야' 등의 'ㅎ' 탈락은 형태소분석 자체가 잘 되지 않아 불규칙 적용하지 않음 # '묻다'는 '땅에 묻다'와 '물어보다'의 의미가 구분되지 않아 불규칙 적용하지 않음 if jong[-1] == 7 and \ (newToken[-1] in ("걷", "싣", "듣") or newToken[-2:] in ("깨닫", "일컫")): # ㄷ받침 newJong = hangul.build(jong[0], jong[1], 8) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) elif jong[-1] == 19 and \ (newToken[-1] in ("긋", "낫", "붓", "잇", "젓", "짓")): # ㅅ받침 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) elif jong[-1] == 17 \ and newToken[-1] not in ("입", "잡", "씹", "좁", "접", "뽑"): # ㅂ받침: 눕+어=누워 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) if token == "어": token = "워" elif token == "아": token = "와" tokenWord = "%s%s" % (newToken, token) formerTokenTag = "EC" new_check = True elif token in {"어야", "아야", "어다", "아다"}: ending = token[-1] if jong[-1] == 0: # 받침 없음 if jong[1] == 20 and token.startswith("어"): # 달려야, 마셔야 newJong = hangul.build(jong[0], 6, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 18 and jong[0] == 5: # '르' 불규칙: 마르+아야=말라야 tmp = hangul.separate(newToken[-2]) newJong = hangul.build(tmp[0], tmp[1], 8) tmp2 = hangul.separate(token[0]) newJong2 = hangul.build(5, tmp2[1], tmp2[2]) if len(newToken) == 2: # 마르, 오르, 바르.. newToken = "%s%s%s" % (newJong, newJong2, ending) else: newToken = "%s%s%s%s" % (newToken[:-2], newJong, newJong2, ending) tokenWord = newToken elif jong[1] == 18 and token.startswith("어"): # 'ㅡ' 불규칙: 쓰+어야=써야 newJong = hangul.build(jong[0], 4, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 13 and token.startswith("어"): # 세우+어야=세워야 newJong = hangul.build(jong[0], 14, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 8 and token.startswith("아"): # 따라오+아야=따라와야 newJong = hangul.build(jong[0], 9, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong == (18,0,0) and (token.startswith("아") or token.startswith("어")): # 해야 if len(newToken) == 1: tokenWord = "해%s" % ending else: tokenWord = "%s해%s" % (newToken[:-1], ending) else: tokenWord = "%s%s" % (newToken, ending) # 펴야, 자야 # 받침 있음 elif jong[-1] != 0: if jong[-1] == 17 \ and newToken[-1] not in ("입", "잡", "씹", "좁", "접", "뽑"): # ㅂ받침: 눕+어=누워 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) if token.startswith("어"): # 아름다워야 tokenWord = "%s워%s" % (newToken, ending) elif token.startswith("아"): # 고와야 tokenWord = "%s와%s" % (newToken, ending) elif jong[-1] == 19 and \ (newToken[-1] in ("긋", "낫", "붓", "잇", "젓", "짓")): # ㅅ받침 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = "%s%s" % (newToken, token) elif jong[-1] == 7 and \ (newToken[-1] in ("걷", "싣", "듣") or newToken[-2:] in ("깨닫", "일컫")): # ㄷ받침 newJong = hangul.build(jong[0], jong[1], 8) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = "%s%s%s" % (newToken, token[0], ending) else: tokenWord = "%s%s" % (newToken, token) formerTokenTag = "EC" new_check = True elif token.endswith("지") and formerTag in {"VV", "VA", "VX"}: # 하다_지_모른다 tokenWord = "%s다_지_" % newToken formerTokenTag = "EC" new_check = True else: formerTokenTag = "REMOVE_EC" new_check = True # verb and adjective elif tag in ["VV", "VA", "VX"]: if formerTokenTag == "EC": if tag == "VX" and token == "가지": # 해가지고, 떠가지고 = 하다, 뜨다 tokenWord = original_token else: tokenWord = "%s%s" % (tokenWord, token+"다") elif formerTokenTag.startswith("NNB"): if formerTokenTag.endswith("1") and token in {"있", "없"}: # 하다_수_있다 tokenWord = "%s_%s다" % (tokenWord, token) elif formerTokenTag.endswith("2") and token in {"모르", "않", "말"}: # 하다_지_모르다 tokenWord = "%s_%s다" % (tokenWord, token) elif formerTokenTag.endswith("3"): # 사랑_때문_자라다 tokenWord = "%s_%s다" % (tokenWord, token) else: tokenWord += "%s다" % token elif formerTokenTag == "JX": if token == "없": tokenWord += "_%s다" % token else: tokenWord += " %s다" % token else: if tag in ("VX", "VA") and token in ["않", "없", "못하", "말", "주"]: tokenWord += "_%s다" % token else: # rest all tag including VX # insert white space if V-V or VA-VA case(grammar error) if formerTokenTag == tag: tokenWord += " %s다" % token else: tokenWord += "%s다" % token formerTokenTag = tag new_check = True # adverb elif tag == "MAG" and token == "못": tokenWord += "_%s" % token elif tag == "MAG" and token == "없이": tokenWord = "%s없다" % tokenWord # JKB: 부사격조사 elif tag == "JKB" and token == "같이": tokenWord = "%s같다" % tokenWord else: idx += 1 formerTag = tag new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue formerTag = tag new_check, formerTokenTag = ending_check(new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) idx += 1 tokenList.append(tokenWord.strip()) # if it's the last word in 'Komoran' if formerTokenTag == "EC": tokenList[-1] = original_token finalToken = " ".join([a for a in tokenList if a not in {"", "하다", "암트다"}]) # stop word if "안 하다" in finalToken: finalToken = finalToken.replace("안 하다", "안하다") for neg in {"없다", "않다", "못하다", "안하다"}: if neg == finalToken: continue if " _%s" % neg in finalToken: finalToken = finalToken.replace(" _%s" % neg, " %s" % neg) if " %s" % neg in finalToken: finalToken = finalToken.replace(" %s" % neg, "_%s" % neg) if (" _%s" % neg) in finalToken: finalToken = finalToken.replace((" _%s" % neg), ("_%s" % neg)) finalToken = finalToken.strip("_").replace("__", "_").replace(" _", " ").replace("_ ", " ") return re.sub(r"\s+", " ", finalToken)
def test_separation(): assert hangul.separate(u'가') == (0, 0, 0) assert hangul.separate(u'까') == (1, 0, 0) assert hangul.separate(u'갸') == (0, 2, 0) assert hangul.separate(u'각') == (0, 0, 1)
def test_separation(): assert hangul.separate(u"가") == (0, 0, 0) assert hangul.separate(u"까") == (1, 0, 0) assert hangul.separate(u"갸") == (0, 2, 0) assert hangul.separate(u"각") == (0, 0, 1)
def komoran_processing(komoran, complex_verb_set): tokenList = list() formerTokenTag = "" original_token = "" word_cnt = len(komoran) word_idx = 0 for phrase in komoran: phrase_cnt = len(phrase) idx = 0 word_idx += 1 tokenWord = "" formerTag = "" new_check = False while idx < phrase_cnt: token = phrase[idx].getFirst().replace( " ", "") #remove white space in proper nouns if " " not in token and len(token) > 9: idx += 1 new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue tag = phrase[idx].getSecond() ########## FIRST PHONEME if idx == 0 or (idx == 1 and formerTag == "XPN"): #XPN: 체언접두사 if formerTokenTag == "REMOVE_EC": tokenList[-1] = "%s " % tokenList[-1] # To combine the former token and the current token elif formerTokenTag == "MAG": tokenWord = tokenList[-1] del tokenList[-1] elif formerTokenTag == "EC": if tag not in {"VV", "VA", "VX" } or (tag in {"VV", "VA", "VX"} and token not in complex_verb_set): tokenList[-1] = original_token formerTokenTag = "" original_token = "" # prefix(체언접두사) if tag == "XPN": tokenWord = token formerTag = tag idx += 1 new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue else: # common noun, proper noun, root adverb and foreign language if tag in {"NNG", "NNP", "SL", "XR", "MAG"}: if tag == "SL": token = token.lower() if tag == "MAG": if token in {"안", "못", "잘못"}: formerTokenTag = "MAG" new_check = True else: idx += 1 new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue elif tag == "NNP": token = "%s " % token elif formerTokenTag == "NNB" and token == "밖": token = "밖에" tokenWord = tokenWord.rstrip("_") formerTokenTag = "JX" new_check = True else: pass # dependent nouns elif tag == "NNB" and token in {"수", "지", "때문"}: if token == "수" and formerTokenTag in { "VV", "VA", "VX" }: formerTokenTag = "NNB1" elif token == "수" and formerTokenTag not in { "VV", "VA", "VX" }: formerTokenTag = "" pass # 사람의 '수' 등의 형태소분석 오류 elif token == "지" and formerTokenTag in { "VV", "VA", "VX" }: formerTokenTag = "NNB2" elif token == "때문": formerTokenTag = "NNB3" else: formerTokenTag = "" new_check = False idx += 1 continue new_check = True try: if formerTokenTag.startswith("NNB"): token = "%s_%s" % (tokenList[-1], token) del tokenList[-1] else: pass except IndexError: # not dependent nouns formerTokenTag = "" new_check = False idx += 1 continue # verb, adjective and auxiliary predicate elif tag in ["VV", "VA", "VX"]: if formerTokenTag == "EC": token = "%s%s" % (tokenList[-1], "%s다" % token) del tokenList[-1] elif formerTokenTag == "JX": if token == "없": # 수밖에_없다 token = "_%s다" % token else: token = " %s다" % token elif formerTokenTag.startswith("NNB"): if formerTokenTag.endswith("1") and token in { "있", "없" }: # 하다_수_있다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] elif formerTokenTag.endswith("2") and token in { "모르", "않", "말" }: # 하다_지_모르다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] elif formerTokenTag.endswith("3"): # 사랑_때문_자라다 tokenWord = tokenList[-1] token = "_%s다" % token del tokenList[-1] else: token = "%s다" % token # only in case negative words elif token in ["않", "없", "못하", "말", "싫", "주"]: try: token = "%s%s" % (tokenList[-1], "_%s다" % token) del tokenList[-1] except IndexError: token = "_%s다" % token else: token = "%s다" % token formerTokenTag = tag new_check = True elif formerTokenTag == "NNB1" and tag == "JX": if token == "밖에": # 수 밖에 = 수밖에 token = "%s%s" % (tokenList[-1].rstrip("_"), token) del tokenList[-1] formerTokenTag = "JX" new_check = True elif token == "도": # 갈 수도 있다 = 가다_수_있다 new_check = True else: # just pass the rest idx += 1 formerTag = tag new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue tokenWord += token formerTag = tag # checking 'formerTokenTag' to be newly assigned new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) ########## FROM SECOND PHONEME else: if formerTokenTag == "REMOVE_EC": tokenWord = "%s " % tokenWord elif formerTokenTag.startswith("NNB") and tag not in { "VV", "VA", "VX" }: tokenWord = "%s " % tokenWord if formerTag in ["ETM", "ETN"]: # ETM: 관성형전성어미, ETN: 명사형전성어미 tokenWord = "%s " % tokenWord # common noun, proper noun, root and foreign language, suffix(noun) and dependent noun if tag in {"NNG", "NNP", "XR", "SL", "NNB", "XSN"}: if tag == "SL": token = token.lower() if tag == "NNP": if formerTag == "NNP": tokenWord = "%s%s " % (tokenWord, token) else: tokenWord = "%s %s " % (tokenWord, token) elif tag == "NNB": if token in {"수", "지", "때문"}: if token == "수": if formerTag in {"VV", "VA", "VX"}: tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB1" else: tokenWord = "%s %s" % (tokenWord.strip(), token) elif token == "지" and formerTag in { "VV", "VA", "VX" }: tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB2" elif token == "때문": tokenWord = "%s_%s" % (tokenWord.strip(), token) formerTokenTag = "NNB3" else: pass new_check = True else: pass else: tokenWord += token # 부정지정사 '아니' elif tag == "VCN": tokenWord = "%s_%s다" % (tokenWord, token) # 수 밖에 = 수밖에 elif (formerTokenTag == "NNB1" or formerTag == "NNB1") and tag == "JX": if token == "밖에": tokenWord = "%s%s" % (tokenWord.rstrip("_"), token) formerTokenTag = "JX" new_check = True elif token == "도": new_check = True # suffix(adjective and verb) elif tag in {"XSA", "XSV"}: token = "%s다" % token tokenWord += token # connective endings elif tag == "EC": original_token = tokenWord newToken = tokenWord.rstrip("다") try: jong = hangul.separate(newToken[-1]) #마지막 글자 분해 except IndexError: idx += 1 formerTag = tag new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue if token in {"아", "어"}: if formerTag == "VCP": idx += 1 formerTag = tag new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue # 받침없음 if jong[-1] == 0: if jong[1] == 20 and token == "어": # 달리+어=달려 newJong = hangul.build(jong[0], 6, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 18 and jong[ 0] == 5: # '르' 불규칙: 마르+아=말라 tmp = hangul.separate(newToken[-2]) newJong = hangul.build(tmp[0], tmp[1], 8) tmp2 = hangul.separate(token) newJong2 = hangul.build(5, tmp2[1], tmp2[2]) if len(newToken) == 2: # 마르, 오르, 바르.. newToken = "%s%s" % (newJong, newJong2) else: newToken = "%s%s%s" % (newToken[:-2], newJong, newJong2) tokenWord = newToken elif jong[ 1] == 18 and token == "어": # 'ㅡ' 불규칙: 쓰+어=써 newJong = hangul.build(jong[0], 4, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 13 and token == "어": # 세우+어=세워 newJong = hangul.build(jong[0], 14, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 8 and token == "아": # 따라오+아=따라와 newJong = hangul.build(jong[0], 9, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = newToken elif jong[1] == 11 and token == "어": # 되+어=되어 newToken = "%s%s" % (newToken, "어") tokenWord = newToken elif jong == (18, 0, 0) and token in {"아", "어"}: if len(newToken) == 1: tokenWord = "해" else: tokenWord = "%s해" % newToken[:-1] else: tokenWord = newToken # 펴, 자 # 받침있음 else: # '빨개야', '파래야' 등의 'ㅎ' 탈락은 형태소분석 자체가 잘 되지 않아 불규칙 적용하지 않음 # '묻다'는 '땅에 묻다'와 '물어보다'의 의미가 구분되지 않아 불규칙 적용하지 않음 if jong[-1] == 7 and \ (newToken[-1] in ("걷", "싣", "듣") or newToken[-2:] in ("깨닫", "일컫")): # ㄷ받침 newJong = hangul.build(jong[0], jong[1], 8) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) elif jong[-1] == 19 and \ (newToken[-1] in ("긋", "낫", "붓", "잇", "젓", "짓")): # ㅅ받침 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) elif jong[-1] == 17 \ and newToken[-1] not in ("입", "잡", "씹", "좁", "접", "뽑"): # ㅂ받침: 눕+어=누워 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) if token == "어": token = "워" elif token == "아": token = "와" tokenWord = "%s%s" % (newToken, token) formerTokenTag = "EC" new_check = True elif token in {"어야", "아야", "어다", "아다"}: ending = token[-1] if jong[-1] == 0: # 받침 없음 if jong[1] == 20 and token.startswith( "어"): # 달려야, 마셔야 newJong = hangul.build(jong[0], 6, jong[-1]) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 18 and jong[ 0] == 5: # '르' 불규칙: 마르+아야=말라야 tmp = hangul.separate(newToken[-2]) newJong = hangul.build(tmp[0], tmp[1], 8) tmp2 = hangul.separate(token[0]) newJong2 = hangul.build(5, tmp2[1], tmp2[2]) if len(newToken) == 2: # 마르, 오르, 바르.. newToken = "%s%s%s" % (newJong, newJong2, ending) else: newToken = "%s%s%s%s" % (newToken[:-2], newJong, newJong2, ending) tokenWord = newToken elif jong[1] == 18 and token.startswith( "어"): # 'ㅡ' 불규칙: 쓰+어야=써야 newJong = hangul.build(jong[0], 4, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 13 and token.startswith( "어"): # 세우+어야=세워야 newJong = hangul.build(jong[0], 14, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong[1] == 8 and token.startswith( "아"): # 따라오+아야=따라와야 newJong = hangul.build(jong[0], 9, jong[-1]) if len(newToken) == 1: newToken = "%s%s" % (newJong, ending) else: newToken = "%s%s%s" % (newToken[:-1], newJong, ending) tokenWord = newToken elif jong == (18, 0, 0) and (token.startswith("아") or token.startswith("어")): # 해야 if len(newToken) == 1: tokenWord = "해%s" % ending else: tokenWord = "%s해%s" % (newToken[:-1], ending) else: tokenWord = "%s%s" % (newToken, ending ) # 펴야, 자야 # 받침 있음 elif jong[-1] != 0: if jong[-1] == 17 \ and newToken[-1] not in ("입", "잡", "씹", "좁", "접", "뽑"): # ㅂ받침: 눕+어=누워 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) if token.startswith("어"): # 아름다워야 tokenWord = "%s워%s" % (newToken, ending) elif token.startswith("아"): # 고와야 tokenWord = "%s와%s" % (newToken, ending) elif jong[-1] == 19 and \ (newToken[-1] in ("긋", "낫", "붓", "잇", "젓", "짓")): # ㅅ받침 newJong = hangul.build(jong[0], jong[1], 0) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = "%s%s" % (newToken, token) elif jong[-1] == 7 and \ (newToken[-1] in ("걷", "싣", "듣") or newToken[-2:] in ("깨닫", "일컫")): # ㄷ받침 newJong = hangul.build(jong[0], jong[1], 8) if len(newToken) == 1: newToken = newJong else: newToken = "%s%s" % (newToken[:-1], newJong) tokenWord = "%s%s%s" % (newToken, token[0], ending) else: tokenWord = "%s%s" % (newToken, token) formerTokenTag = "EC" new_check = True elif token.endswith("지") and formerTag in { "VV", "VA", "VX" }: # 하다_지_모른다 tokenWord = "%s다_지_" % newToken formerTokenTag = "EC" new_check = True else: formerTokenTag = "REMOVE_EC" new_check = True # verb and adjective elif tag in ["VV", "VA", "VX"]: if formerTokenTag == "EC": if tag == "VX" and token == "가지": # 해가지고, 떠가지고 = 하다, 뜨다 tokenWord = original_token else: tokenWord = "%s%s" % (tokenWord, token + "다") elif formerTokenTag.startswith("NNB"): if formerTokenTag.endswith("1") and token in { "있", "없" }: # 하다_수_있다 tokenWord = "%s_%s다" % (tokenWord, token) elif formerTokenTag.endswith("2") and token in { "모르", "않", "말" }: # 하다_지_모르다 tokenWord = "%s_%s다" % (tokenWord, token) elif formerTokenTag.endswith("3"): # 사랑_때문_자라다 tokenWord = "%s_%s다" % (tokenWord, token) else: tokenWord += "%s다" % token elif formerTokenTag == "JX": if token == "없": tokenWord += "_%s다" % token else: tokenWord += " %s다" % token else: if tag in ("VX", "VA") and token in [ "않", "없", "못하", "말", "주" ]: tokenWord += "_%s다" % token else: # rest all tag including VX # insert white space if V-V or VA-VA case(grammar error) if formerTokenTag == tag: tokenWord += " %s다" % token else: tokenWord += "%s다" % token formerTokenTag = tag new_check = True # adverb elif tag == "MAG" and token == "못": tokenWord += "_%s" % token elif tag == "MAG" and token == "없이": tokenWord = "%s없다" % tokenWord # JKB: 부사격조사 elif tag == "JKB" and token == "같이": tokenWord = "%s같다" % tokenWord else: idx += 1 formerTag = tag new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) continue formerTag = tag new_check, formerTokenTag = ending_check( new_check, formerTokenTag, idx, word_idx, phrase_cnt, word_cnt) idx += 1 tokenList.append(tokenWord.strip()) # if it's the last word in 'Komoran' if formerTokenTag == "EC": tokenList[-1] = original_token finalToken = " ".join([a for a in tokenList if a not in {"", "하다", "암트다"}]) # stop word if "안 하다" in finalToken: finalToken = finalToken.replace("안 하다", "안하다") for neg in {"없다", "않다", "못하다", "안하다"}: if neg == finalToken: continue if " _%s" % neg in finalToken: finalToken = finalToken.replace(" _%s" % neg, " %s" % neg) if " %s" % neg in finalToken: finalToken = finalToken.replace(" %s" % neg, "_%s" % neg) if (" _%s" % neg) in finalToken: finalToken = finalToken.replace((" _%s" % neg), ("_%s" % neg)) finalToken = finalToken.strip("_").replace("__", "_").replace(" _", " ").replace( "_ ", " ") return re.sub(r"\s+", " ", finalToken)
def test_separation(): assert hangul.separate(u'가') == (0, 0, 0) assert hangul.separate(u'까') == (1, 0, 0) assert hangul.separate(u'갸') == (0, 2, 0) assert hangul.separate(u'각') == (0, 0, 1)