def _convert(match_char):
c = match_char.group(0)
try:
return unicodedata.name(c)
except ValueError:
# Throws a ValueError if the name doesn't exist.
return ''
def is_hiragana(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "HIRAGANA" in character_name
def is_katakana(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "KATAKANA" in character_name
def is_hangul(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "HANGUL" in character_name
def is_thai(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "THAI" in character_name
def is_cjk(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError:
return False
return "CJK" in character_name
def random_emoji():
emoji_decimal = random.choice(emojis)
emoji_escaped = b"\\U%08x" % emoji_decimal
emoji_char = emoji_escaped.decode(
'unicode-escape'
) # python2 work-around is to decode with unicode-escape
emoji_name = unicodedata2.name(emoji_char, NO_NAME_ERROR).capitalize()
return (emoji_char, emoji_escaped, emoji_name)
def is_accentuated(character: str) -> bool:
try:
description: str = unicodedata.name(character)
except ValueError:
return False
return ("WITH GRAVE" in description or "WITH ACUTE" in description
or "WITH CEDILLA" in description or "WITH DIAERESIS" in description
or "WITH CIRCUMFLEX" in description or "WITH TILDE" in description)
def main():
for i in xrange(0, MAX_UNICODE_CHAR):
s = r'\U' + ('%08X' % i)
t = eval('u"' + s + '"')
name = unicodedata.name(t, None)
fall = unidecode.unidecode(t)
if name is None: continue
name = name.encode('utf8')
print '%08x\t%08d\t%s\t%s\t%s\t%s' % (i, i, s, t, fall, name)
def unicode_pipe(text):
'''Replaces unicode characters with their official names.'''
out = []
for c in text:
try:
out.append(unicodedata2.name(c))
except:
out.append('UNKNOWN CHARACTER (%s)' % c)
return ', '.join(out)
def appendResults_(self, maxResults):
start = len(self.w.unicodeList)
unicodeItems = [
dict(char=chr(uni),
unicode=f"U+{uni:04X}",
name=unicodedata.name(chr(uni), ""))
for uni in self.searchResults[start:start + maxResults]
]
if len(self.searchResults) > start + maxResults:
unicodeItems.append(dict(name="...more..."))
self.w.unicodeList.extend(unicodeItems)
def __getitem__(self, charCode): try: # use unicodedata backport to python2, if available: # https://github.com/mikekap/unicodedata2 import unicodedata2 as unicodedata except ImportError: import unicodedata try: return unicodedata.name(unichr(charCode)) except ValueError: return "????"
def __getitem__(self, charCode):
try:
# use unicodedata backport to python2, if available:
# https://github.com/mikekap/unicodedata2
import unicodedata2 as unicodedata
except ImportError:
import unicodedata
try:
return unicodedata.name(unichr(charCode))
except ValueError:
return "????"
def getNameValue(self, lang): if lang == 'en': try: return unicodedata.name(self.text) except ValueError: return STR_NO_CHAR_ERROR if not unicodeInfo.unicodeData[lang]: return STR_NO_FILE_ERROR try: return unicodeInfo.unicodeData[lang][self.num][0] except KeyError: return STR_NO_CHAR_ERROR
def demoji_pipe(text):
'''Replaces emoji in text with their official names.'''
out = []
for c in text:
if c in emoji.UNICODE_EMOJI:
try:
out.append(unicodedata2.name(c) + ' ')
except:
out.append('(UNKNOWN)')
else:
out.append(c)
return ''.join(out)
def check_types(Langs):
for iso, lang in Langs.items():
if "includes" in lang:
if not check_is_valid_list(lang["includes"]):
logging.error("'%s' has invalid list 'includes'" % iso)
if "source" in lang:
if not check_is_valid_list(lang["source"]):
logging.error("'%s' has invalid list 'source'" % iso)
if "orthographies" in lang:
if not check_is_valid_list(lang["orthographies"]):
logging.error("'%s' has invalid list 'orthographies'" % iso)
for o in lang["orthographies"]:
if "base" in o:
if iso == "arg":
for i, c in enumerate(list(o["base"].replace(" ",
""))):
if unicodedata2.category(c).startswith("Z"):
logging.error("'%s' has invalid whitespace "
"characters '%s' at %d" %
(iso, unicodedata2.name(c), i))
if not check_is_valid_glyph_string(o["base"]):
logging.error("'%s' has invalid 'base' glyph list" %
iso)
if "combinations" in o:
if not check_is_valid_combation_string(o["combinations"]):
logging.error("'%s' has invalid 'combination' string" %
iso)
if "name" not in lang and "preferred_name" not in lang:
logging.error("'%s' has neither 'name' nor 'preferred_name'" % iso)
if "name" in lang and "preferred_name" in lang and \
lang["name"] == lang["preferred_name"]:
logging.error("'%s' has 'name' and 'preferred_name', but they are "
"identical" % iso)
# if "todo_status" in lang and lang["todo_status"] not in VALID_TODOS:
# logging.error("'%s' has an invalid 'todo_status'" % iso)
if "status" in lang and lang["status"] not in VALID_STATUS:
logging.error("'%s' has an invalid 'status'" % iso)
def name(char, *args):
"""Returns the name of a character.
Raises a ValueError exception if the character is undefined, unless an
extra argument is given, in which case it will return that argument.
"""
if type(char) is int:
char = unichr(char)
# First try and get the name from unidata, which is faster and supports
# CJK and Hangul automatic names
try:
return unicodedata.name(char)
except ValueError as val_error:
load_data()
if ord(char) in _character_names_data:
return _character_names_data[ord(char)]
elif args:
return args[0]
else:
raise val_error
def name(char, *args):
"""Returns the name of a character.
Raises a ValueError exception if the character is undefined, unless an
extra argument is given, in which case it will return that argument.
"""
if type(char) is int:
char = unichr(char)
# First try and get the name from unidata, which is faster and supports
# CJK and Hangul automatic names
try:
return unicodedata.name(char)
except ValueError as val_error:
load_data()
if ord(char) in _character_names_data:
return _character_names_data[ord(char)]
elif args:
return args[0]
else:
raise val_error
def to_string(c):
digit = format(ord(c), 'x')
name = unicodedata.name(c, 'Name not found.')
return fmt.format(digit, name, c)
def is_latin(character: str) -> bool:
try:
description = unicodedata.name(character) # type: str
except ValueError:
return False
return "LATIN" in description
def is_accentuated(character: str) -> bool:
try:
description = unicodedata.name(character) # type: str
except ValueError:
return False
return "WITH GRAVE" in description or "WITH ACUTE" in description or "WITH CEDILLA" in description
"""
Helper script to compare the previously GF-published Rasa with the current to
check which encoded glyphs got added
"""
import unicodedata2
from fontTools.ttLib import TTFont
before = TTFont("tests/Rasa-Regular-GF-2020-12.ttf")
now = TTFont("fonts/Rasa/Rasa-Regular.ttf")
before_unicodes = set(before["cmap"].getBestCmap().keys())
now_unicodes = set(now["cmap"].getBestCmap().keys())
new = sorted(now_unicodes.difference(before_unicodes))
for n in new:
print("%s # %s (%s)" % ("U+" + f"{n:04X}", now["cmap"].getBestCmap()[n],
unicodedata2.name(chr(n))))
def check_types(Langs):
for iso, lang in Langs.items():
if "includes" in lang:
if not check_is_valid_list(lang["includes"]):
log.error("'%s' has invalid list 'includes'" % iso)
if "source" in lang:
if not check_is_valid_list(lang["source"]):
log.error("'%s' has invalid list 'source'" % iso)
if "orthographies" in lang:
if not check_is_valid_list(lang["orthographies"]):
log.error("'%s' has invalid list 'orthographies'" % iso)
for o in lang["orthographies"]:
if "base" in o:
if iso == "arg":
chars = list(o["base"].replace(" ", ""))
for i, c in enumerate(chars):
if unicodedata2.category(c).startswith("Z"):
log.error("'%s' has invalid whitespace "
"characters '%s' at %d" %
(iso, unicodedata2.name(c), i))
if not check_is_valid_glyph_string(o["base"], iso):
log.error("'%s' has invalid 'base' glyph list" % iso)
if "auxiliary" in o:
if not check_is_valid_glyph_string(o["auxiliary"], iso):
log.error("'%s' has invalid 'auxiliary' glyph list" %
iso)
allowed = [
"autonym",
"inherit",
"script",
"base",
"marks",
"auxiliary",
"numerals",
"status",
"note",
"punctuation", # tolerated for now, but unused
"preferred_as_group",
"design_note"
]
invalid = [k for k in o.keys() if k not in allowed]
if len(invalid):
log.warn("'%s' has invalid orthography keys: '%s'" %
(iso, "', '".join(invalid)))
if "status" not in o:
log.error("'%s' has an orthography (script '%s') that is "
"missing 'status'" % (iso, o["script"]))
else:
if o["status"] not in ORTHOGRAPHY_STATUSES:
log.error("'%s' has an orthography status '%s' which "
"is invalid, should be one of %s" %
(iso, o["status"],
", ".join(ORTHOGRAPHY_STATUSES)))
primary_orthography = [
o for o in lang["orthographies"]
if "status" in o and o["status"] == "primary"
]
if len(primary_orthography) == 0:
log.error("'%s' has no primary orthography" % iso)
if "name" not in lang and "preferred_name" not in lang:
log.error("'%s' has neither 'name' nor 'preferred_name'" % iso)
if "name" in lang and "preferred_name" in lang and \
lang["name"] == lang["preferred_name"]:
log.error("'%s' has 'name' and 'preferred_name', but they are "
"identical" % iso)
if "status" in lang and lang["status"] not in STATUSES:
log.error("'%s' has an invalid 'status'" % iso)
if "validity" not in lang:
log.warn("'%s' is missing 'validity'" % iso)
if "validity" in lang and lang["validity"] not in VALIDITYLEVELS:
log.error("'%s' has invalid 'validity'" % iso)
if "speakers" in lang:
if (re.search(r"[^\d]", str(lang["speakers"]))):
log.error("'%s' has invalid 'speakers' '%s' - only numbers "
"are allowed" % (iso, lang["speakers"]))
