def hangle_test(): from soynlp.hangle import normalize from soynlp.hangle import compose from soynlp.hangle import decompose from soynlp.hangle import character_is_korean from soynlp.hangle import character_is_jaum from soynlp.hangle import character_is_moum from soynlp.hangle import to_base from soynlp.hangle import levenshtein from soynlp.hangle import jamo_levenshtein normalized_ = normalize('123이건테스트ab테스트') if not (normalized_ == '이건테스트 테스트'): raise ValueError('{} should be 이건테스트 테스트'.format(normalized_)) if not (('ㄱ', 'ㅏ', 'ㄴ') == decompose('간')): raise ValueError('decompose("간") -> {}'.format(decompose('간'))) if not ((' ', 'ㅗ', ' ') == decompose('ㅗ')): raise ValueError('decompose("ㅗ") -> {}'.format(decompose('ㅗ'))) if not (('ㅋ', ' ', ' ') == decompose('ㅋ')): raise ValueError('decompose("ㅋ") -> {}'.format(decompose('ㅋ'))) if not ('감' == compose('ㄱ', 'ㅏ', 'ㅁ')): raise ValueError("compose('ㄱ', 'ㅏ', 'ㅁ') -> {}".format(compose('ㄱ', 'ㅏ', 'ㅁ'))) if not character_is_korean('감'): raise ValueError('character_is_korean("감") -> {}'.format(character_is_korean('감'))) if character_is_korean('a'): raise ValueError('character_is_korean("a") -> {}'.format(character_is_korean('a'))) if not character_is_jaum('ㅋ'): raise ValueError('character_is_jaum("ㅋ") -> {}'.format(character_is_jaum('ㅋ'))) if character_is_jaum('a'): raise ValueError('character_is_jaum("a") -> {}'.format(character_is_jaum('a'))) if not character_is_moum('ㅗ'): raise ValueError('character_is_jaum("ㅗ") -> {}'.format(character_is_jaum('ㅗ'))) if character_is_moum('a'): raise ValueError('character_is_jaum("a") -> {}'.format(character_is_jaum('a'))) if not (to_base('ㄱ') == 12593): raise ValueError('to_base("ㄱ") -> {}'.format(to_base('ㄱ'))) if 1 != levenshtein('가나', '가남'): raise ValueError("levenshtein('가나', '가남') -> {}".format(levenshtein('가나', '가남'))) if 0.1 != levenshtein('가나', '가남', {('나', '남'):0.1}): raise ValueError("levenshtein('가나', '가남', {('나', '남'):0.1}) -> {}".format(levenshtein('가나', '가남', {('나', '남'):0.1}))) if 1/3 != jamo_levenshtein('가나', '가남'): raise ValueError("jamo_levenshtein('가나', '가남') -> {}".format(jamo_levenshtein('가나', '가남'))) print('all hangle tests have been successed\n\n')
def jamo_to_word(jamo): jamo_list, idx = [], 0 while idx < len(jamo): if not character_is_korean(jamo[idx]): jamo_list.append(jamo[idx]) idx += 1 else: jamo_list.append(jamo[idx:idx + 3]) idx += 3 word = "" for jamo_char in jamo_list: if len(jamo_char) == 1: word += jamo_char elif jamo_char[2] == "-": word += compose(jamo_char[0], jamo_char[1], " ") else: word += compose(jamo_char[0], jamo_char[1], jamo_char[2]) return word
def jamo_sentence(sent): def transform(char): if char == ' ': return char cjj = decompose(char) if len(cjj) == 1: return cjj cjj_ = ''.join(c if c != ' ' else '-' for c in cjj) return cjj_ sent_ = [] for char in sent: if character_is_korean(char): sent_.append(transform(char)) else: sent_.append(char) sent_ = doublespace_pattern.sub(' ', ''.join(sent_)) return sent_
def sent_to_jamo(sent): def transform(char): if char == " ": return char cjj = decompose(char) if len(cjj) == 1: return cjj cjj_ = "".join(c if c != " " else "-" for c in cjj) return cjj_ sent_ = [] for char in sent: if character_is_korean(char): sent_.append(transform(char)) else: sent_.append(char) doublespace_pattern = re.compile("\s+") sent_ = doublespace_pattern.sub(" ", "".join(sent_)) return sent_
def hangle_test(): from soynlp.hangle import normalize from soynlp.hangle import compose from soynlp.hangle import decompose from soynlp.hangle import character_is_korean from soynlp.hangle import character_is_jaum from soynlp.hangle import character_is_moum from soynlp.hangle import to_base from soynlp.hangle import levenshtein from soynlp.hangle import jamo_levenshtein normalized_ = normalize('123이건테스트ab테스트') if not (normalized_ == '이건테스트 테스트'): raise ValueError('{} should be 이건테스트 테스트'.format(normalized_)) if not (('ㄱ', 'ㅏ', 'ㄴ') == decompose('간')): raise ValueError('decompose("간") -> {}'.format(decompose('간'))) if not ((' ', 'ㅗ', ' ') == decompose('ㅗ')): raise ValueError('decompose("ㅗ") -> {}'.format(decompose('ㅗ'))) if not (('ㅋ', ' ', ' ') == decompose('ㅋ')): raise ValueError('decompose("ㅋ") -> {}'.format(decompose('ㅋ'))) if not ('감' == compose('ㄱ', 'ㅏ', 'ㅁ')): raise ValueError("compose('ㄱ', 'ㅏ', 'ㅁ') -> {}".format( compose('ㄱ', 'ㅏ', 'ㅁ'))) if not character_is_korean('감'): raise ValueError('character_is_korean("감") -> {}'.format( character_is_korean('감'))) if character_is_korean('a'): raise ValueError('character_is_korean("a") -> {}'.format( character_is_korean('a'))) if not character_is_jaum('ㅋ'): raise ValueError('character_is_jaum("ㅋ") -> {}'.format( character_is_jaum('ㅋ'))) if character_is_jaum('a'): raise ValueError('character_is_jaum("a") -> {}'.format( character_is_jaum('a'))) if not character_is_moum('ㅗ'): raise ValueError('character_is_jaum("ㅗ") -> {}'.format( character_is_jaum('ㅗ'))) if character_is_moum('a'): raise ValueError('character_is_jaum("a") -> {}'.format( character_is_jaum('a'))) if not (to_base('ㄱ') == 12593): raise ValueError('to_base("ㄱ") -> {}'.format(to_base('ㄱ'))) if 1 != levenshtein('가나', '가남'): raise ValueError("levenshtein('가나', '가남') -> {}".format( levenshtein('가나', '가남'))) if 0.1 != levenshtein('가나', '가남', {('나', '남'): 0.1}): raise ValueError( "levenshtein('가나', '가남', {('나', '남'):0.1}) -> {}".format( levenshtein('가나', '가남', {('나', '남'): 0.1}))) if 1 / 3 != jamo_levenshtein('가나', '가남'): raise ValueError("jamo_levenshtein('가나', '가남') -> {}".format( jamo_levenshtein('가나', '가남'))) print('all hangle tests have been successed\n')