def test_aliases(self):
# Check that the aliases defined in the NameAliases.txt file work.
# This should be updated when new aliases are added or the file
# should be downloaded and parsed instead. See #12753.
aliases = [
('LATIN CAPITAL LETTER GHA', 0x01A2),
('LATIN SMALL LETTER GHA', 0x01A3),
('KANNADA LETTER LLLA', 0x0CDE),
('LAO LETTER FO FON', 0x0E9D),
('LAO LETTER FO FAY', 0x0E9F),
('LAO LETTER RO', 0x0EA3),
('LAO LETTER LO', 0x0EA5),
('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0),
('YI SYLLABLE ITERATION MARK', 0xA015),
('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18),
('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5)
]
for alias, codepoint in aliases:
self.checkletter(alias, chr(codepoint))
name = unicodedata.name(chr(codepoint))
self.assertNotEqual(name, alias)
self.assertEqual(unicodedata.lookup(alias),
unicodedata.lookup(name))
with self.assertRaises(KeyError):
unicodedata.ucd_3_2_0.lookup(alias)
def _CleanUpSimpleRegex(regex):
regex = _RemoveChars(regex, [unicodedata.lookup('SPACE'), unicodedata.lookup('EM DASH'), unicodedata.lookup('EN DASH')])
# TODO(shreevatsa): Make this work. Why does this regex have to be simple?
# regex = regex.replace('4', '(LLLL|GLL|LGL|LLG|GG)')
regex = regex.replace('4', '')
assert re.match(r'^[LG.]*$', regex), regex
return regex
def katakana_to_hiragana(text):
# Only a to n, other symbols are not changed
h_sa = unicodedata.lookup('HIRAGANA LETTER SMALL A')
k_sa = unicodedata.lookup('KATAKANA LETTER SMALL A')
k_n = unicodedata.lookup('KATAKANA LETTER N')
return ''.join(chr(ord(h_sa) + ord(c) - ord(k_sa))
if is_katakana(c) and ord(c) <= ord(k_n)
else c for c in text)
def test_nbsp_and_space_glyphs_width(self):
""" Nbsp and space glyphs should have the same width"""
space = 0
nbsp = 0
for x in self.font.glyphs():
if x.unicode == ord(unicodedata.lookup('NO-BREAK SPACE')):
nbsp = x.width
elif x.unicode == ord(unicodedata.lookup('SPACE')):
space = x.width
self.assertEqual(space, nbsp)
def init_view_objects(self):
box = wx.BoxSizer(wx.VERTICAL)
instructions = wx.StaticText(
self.frame,
label="Use %s and %s to move up and down the test cases" % (
unicodedata.lookup('LEFTWARDS ARROW'),
unicodedata.lookup('RIGHTWARDS ARROW')),
style=wx.ALIGN_CENTER,
)
box.Add(instructions, 0, wx.EXPAND)
self.view_objects = MyGLWindow(self.frame, size=(1280, 800))
self.view_objects.SetFocus()
box.Add(self.view_objects, wx.EXPAND, wx.EXPAND)
self.frame.SetSizer(box)
box.SetSizeHints(self.frame)
def get_special_chars(language, additional='', source=''):
"""Return list of special characters."""
for char in settings.SPECIAL_CHARS:
yield format_char(char)
code = language.code.replace('_', '-').split('-')[0]
if code in EXTRA_CHARS:
for char in EXTRA_CHARS[code]:
yield format_char(char)
yield get_quote(code, DOUBLE_OPEN, _('Opening double quote'))
yield get_quote(code, DOUBLE_CLOSE, _('Closing double quote'))
yield get_quote(code, SINGLE_OPEN, _('Opening single quote'))
yield get_quote(code, SINGLE_CLOSE, _('Closing single quote'))
if code in HYPHEN_LANGS:
yield _('Hyphen'), '-', '-'
if code in EN_DASH_LANGS:
yield _('En dash'), '–', '–'
if code in EM_DASH_LANGS:
yield _('Em dash'), '—', '—'
for char in additional:
yield _('User configured character: {}').format(char), char, char
rtl = language.direction == 'rtl'
for char in set(source):
try:
name = unicodedata.name(char)
except ValueError:
continue
if 'ARROW' in name:
if rtl and 'LEFT' in name:
try:
char = unicodedata.lookup(name.replace('LEFT', 'RIGHT'))
except KeyError:
continue
yield format_char(char)
elif rtl and 'RIGHT' in name:
try:
char = unicodedata.lookup(name.replace('RIGHT', 'LEFT'))
except KeyError:
continue
yield format_char(char)
else:
yield format_char(char)
def test_cjk(self):
import sys
if sys.maxunicode < 0x10ffff:
skip("requires a 'wide' python build.")
import unicodedata
cases = ((0x3400, 0x4DB5),
(0x4E00, 0x9FA5))
if unicodedata.unidata_version >= "4.1":
cases = ((0x3400, 0x4DB5),
(0x4E00, 0x9FBB),
(0x20000, 0x2A6D6))
for first, last in cases:
# Test at and inside the boundary
for i in (first, first + 1, last - 1, last):
charname = 'CJK UNIFIED IDEOGRAPH-%X'%i
assert unicodedata.name(unichr(i)) == charname
assert unicodedata.lookup(charname) == unichr(i)
# Test outside the boundary
for i in first - 1, last + 1:
charname = 'CJK UNIFIED IDEOGRAPH-%X'%i
try:
unicodedata.name(unichr(i))
except ValueError:
pass
raises(KeyError, unicodedata.lookup, charname)
def inflection(word):
word = re.sub('que$', '', word)
for a, b in replacements:
word = word.replace(unicodedata.lookup(a), b)
retval = []
for lemma, morphs in latinus.analysesDe(word.lower()).items.iteritems():
for morph in morphs:
temp = {}
try:
for a, b in {#'lemma' : lemma,
'type' : morph.lemme.classe_gr(),
'number' : latinus.Nombre(morph.des.nombre),
'tense' : latinus.Temps(morph.des.temps),
'person' : latinus.Personne(morph.des.personne),
'voice' : latinus.Voix(morph.des.voix),
'gender' : latinus.Genre(morph.des.genre),
'case' : latinus.Cas(morph.des.cas),
#'model' : latinus.Modele(morph.des.modele),
'mood' : latinus.Mode(morph.des.mode),
}.iteritems():
if not re.match('^\s*$', b):
if a == "lemma":
temp[a] = b
else:
try:
temp[a] = french_replacements[a][b]
except:
sys.stderr.write("%s %s\n" % (a, b))
except:
continue
retval.append(temp)
return retval
def _greekletters(letterlist):
for l in letterlist:
ucharname = l.upper()
if (ucharname == 'LAMBDA'):
ucharname = 'LAMDA'
smallname = "GREEK SMALL LETTER "+ucharname;
if (ucharname == 'EPSILON'):
smallname = "GREEK LUNATE EPSILON SYMBOL"
if (ucharname == 'PHI'):
smallname = "GREEK PHI SYMBOL"
_default_macro_list.append(
(l, unicodedata.lookup(smallname))
);
_default_macro_list.append(
(l[0].upper()+l[1:], unicodedata.lookup("GREEK CAPITAL LETTER "+ucharname))
);
def kata2hira(kata):
hira_string = ""
for char in kata:
char_string = unicodedata.name(char)
char_string = char_string.split(" ")
if "SMALL" in char_string:
char_name = char_string[-2:]
hira_name = "HIRAGANA LETTER %s %s" % (char_name[0], char_name[1])
hira_char = unicodedata.lookup(hira_name)
hira_string += hira_char
else:
char_name = char_string[-1:]
hira_name = "HIRAGANA LETTER %s" % char_name[0]
hira_char = unicodedata.lookup(hira_name)
hira_string += hira_char
return hira_string
def test_cjk(self):
import sys
import unicodedata
cases = ((0x3400, 0x4DB5), (0x4E00, 0x9FA5))
if unicodedata.unidata_version >= "5": # don't know the exact limit
cases = ((0x3400, 0x4DB5), (0x4E00, 0x9FCB), (0x20000, 0x2A6D6), (0x2A700, 0x2B734))
elif unicodedata.unidata_version >= "4.1":
cases = ((0x3400, 0x4DB5), (0x4E00, 0x9FBB), (0x20000, 0x2A6D6))
for first, last in cases:
# Test at and inside the boundary
for i in (first, first + 1, last - 1, last):
charname = "CJK UNIFIED IDEOGRAPH-%X" % i
char = ("\\U%08X" % i).decode("unicode-escape")
assert unicodedata.name(char) == charname
assert unicodedata.lookup(charname) == char
# Test outside the boundary
for i in first - 1, last + 1:
charname = "CJK UNIFIED IDEOGRAPH-%X" % i
char = ("\\U%08X" % i).decode("unicode-escape")
try:
unicodedata.name(char)
except ValueError, e:
assert e.message == "no such name"
raises(KeyError, unicodedata.lookup, charname)
def make_basic_glyphs(font):
# This works, but prints out "failed to parse color" 6 times per glyph.
# That is going to be annoying as heck unless I can suppress that output.
for d in config['directories'].values():
for f in os.listdir(d):
fullpath = path.join(d, f)
# This avoids accidentally processing subdirectories. If I ever
# want to change the directory structure drastically, then I can
# investigate os.walk().
if path.isfile(fullpath):
print('Processing file: {}'.format(f))
# Retrieve the filename sans extension, i.e., the glyph's
# unicode name.
glyphname = path.splitext(path.basename(f))[0]
try:
glyphnum = ord(unicodedata.lookup(glyphname))
except KeyError:
printerr(('Filename `{}` does not correspond to a '
'unicode name').format(fullpath),
level='Warning')
continue
glyph = font.createChar(glyphnum)
glyph.importOutlines(fullpath)
glyph.correctDirection()
adjust_bearings(d, glyph, glyphname)
def clean_headword(line, content):
head = clean_head.sub('', line).strip()
content = u'%s, %s' % (head, content)
head = re.sub(u' -a -um', '', head)
head = re.sub(u'^(-)|_|—', '', head).lstrip()
head = re.sub(u' \([^)]+\)$', '', head)
newhead = ''
for char in head:
charname = ud.name(char)
if 'LATIN' in charname:
newcharname = re.search('LATIN (CAPITAL|SMALL) LETTER [\w]+', charname).group(0)
else:
newcharname = charname
newhead += ud.lookup(newcharname)
assert isinstance(newhead, unicode)
#newhead = newhead.decode('utf-8')
if ' ' in newhead.strip() and dosplit(newhead):
newhead = newhead.split(' ')
elif ' ' in newhead.strip() and dontsplit(newhead) and not dosplit(newhead):
newhead = [newhead.split(' ')[0]]
elif '/' in newhead and dontsplit(newhead):
newhead = newhead.split('/')
else:
newhead = newhead.split(';')
return newhead, content
def test_ascii_letters(self):
import unicodedata
for char in "".join(map(chr, xrange(ord("a"), ord("z")))):
name = "LATIN SMALL LETTER %s" % char.upper()
code = unicodedata.lookup(name)
self.assertEqual(unicodedata.name(code), name)
def setup(app):
# Register Builders
app.add_builder(builder.SlideBuilder)
app.add_builder(builder.DirectorySlideBuilder)
app.add_builder(builder.SingleFileSlideBuilder)
app.add_builder(builder.InlineSlideBuilder)
app.add_builder(builder.DirectoryInlineSlideBuilder)
app.connect('html-collect-pages', slides.get_extra_pages)
# core slide configuration
app.add_config_value('slide_title', None, 'html')
app.add_config_value('slide_theme', 'slides', 'html')
app.add_config_value('slide_levels', 3, 'html')
app.add_config_value('slide_theme_options', {}, 'html')
app.add_config_value('slide_theme_path', [], 'html')
app.add_config_value('slide_numbers', False, 'html')
app.add_config_value('slide_footer', None, 'html')
app.add_config_value('autoslides', True, 'env')
# support for linking html output to slides
app.add_config_value('slide_link_html_to_slides', False, 'html')
app.add_config_value('slide_link_html_sections_to_slides', False, 'html')
app.add_config_value('slide_relative_path', '../slides/', 'html')
app.add_config_value('slide_html_slide_link_symbol',
unicodedata.lookup('section sign'), 'html')
# support for linking from slide output to html
app.add_config_value('slide_link_to_html', False, 'html')
app.add_config_value('slide_html_relative_path', '../html/', 'html')
# slide-related directives
app.add_node(directives.if_slides)
app.add_directive('ifnotslides', directives.IfBuildingSlides)
app.add_directive('ifslides', directives.IfBuildingSlides)
app.add_directive('notslides', directives.IfBuildingSlides)
app.add_directive('slides', directives.IfBuildingSlides)
app.add_transform(directives.TransformSlideConditions)
app.add_node(directives.slideconf,
html=(directives.raiseSkip, None),
latex=(directives.raiseSkip, None),
text=(directives.raiseSkip, None),
man=(directives.raiseSkip, None),
texinfo=(directives.raiseSkip, None),
)
app.add_directive('slideconf', directives.SlideConf)
app.connect('doctree-resolved', directives.process_slideconf_nodes)
app.add_node(directives.slide)
app.add_directive('slide', directives.SlideDirective)
app.connect('doctree-resolved', directives.process_slide_nodes)
app.add_node(directives.nextslide)
app.add_directive('nextslide', directives.NextSlideDirective)
app.add_transform(directives.TransformNextSlides)
app.connect('builder-inited', html.inspect_config)
app.connect('html-page-context', html.add_link)
def split_symbols(tokens, local_dict, global_dict):
result = []
for token_number, token_value in tokens:
if token_number == token.NAME and not is_function(token_value, local_dict, global_dict):
try:
unicodedata.lookup('GREEK SMALL LETTER ' + token_value)
result.append((token_number, token_value))
except KeyError:
for char in token_value:
result.append((token.NAME, char))
else:
result.append((token_number, token_value))
return result
def _init_cache():
'''Creates a mapping of wide->narrow and narrow->wide characters'''
global _wide_to_narrow
global _narrow_to_wide
_wide_to_narrow = {}
_narrow_to_wide = {}
char_names = {six.unichr(i): unicodedata.name(six.unichr(i), None)
for i in range(0, 65536)
}
for wide_ch, name in char_names.items():
if name is None:
continue
if name.upper().startswith('FULLWIDTH '):
half_name = name[len('FULLWIDTH '):]
else:
half_name = 'HALFWIDTH {}'.format(name)
try:
half_ch = unicodedata.lookup(half_name)
except KeyError:
pass
else:
_wide_to_narrow[wide_ch] = half_ch
_narrow_to_wide[half_ch] = wide_ch
logger.debug('Mapped %d characters from wide<->narrow',
len(_wide_to_narrow))
def main():
parser = argparse.ArgumentParser(description="""""")
parser.add_argument('--preds')
parser.add_argument('--feats')
args = parser.parse_args()
nam=re.compile("name=([\w_]+)")
acc=""
for linepred, linefeats in zip(open(args.preds,encoding="utf-8").readlines(),open(args.feats,encoding="utf-8").readlines()):
linepred = linepred.strip()
if linepred:
linepred = linepred.split()
if nam.search(linefeats).group(0):
try:
ud_name = nam.search(linefeats).group(0).replace("name=","").replace("_"," ")
except:
ud_name = "HYPHEN-MINUS"
pred = linepred[2][2:-1] #b'2' --> 2
if pred == "1":
acc+=" "
try:
acc+=ud.lookup(ud_name)
except:
acc+="-"
else:
#"end of line"
for x in acc.split(" "):
print(x)
print()
#print(acc.strip())
acc=""
if acc:
for x in acc.split(" "):
print(x)
print()
def greek(name):
''' Returns unicode for greek letters
Example:
>>> print greek('nu')
>>> print greek('lamda')
'''
return _unicodedata.lookup('GREEK SMALL LETTER '+name.upper())
def unicode_repl(match):
s = match.group()
s = s[17:(len(s) - 15)]
for pat, sub in unicode_repl_dict.items():
s = s.replace(pat, sub)
s = unicodedata.lookup(s)
return s
def add(beta):
# characters with an irregular name
IRREGULAR = {
'GREEK CAPITAL LETTER DIGAMMA': 'GREEK LETTER DIGAMMA',
'GREEK SMALL LETTER LUNATE SIGMA': 'GREEK LUNATE SIGMA SYMBOL',
'GREEK CAPITAL LETTER LUNATE SIGMA': 'GREEK CAPITAL LUNATE SIGMA SYMBOL',
}
is_capital = '*' in beta
low_beta = beta.replace('*', '')
letter = re.search('|'.join(re.escape(x) for x in sorted(LETTERS, key=lambda l: -len(l)) ), low_beta).group(0)
diacs = [ DIACRITICS[x][is_capital] for x in low_beta.replace(letter, '') ]
# the only precedence difference between unicode names and betacode
if 'DIALYTIKA' in diacs:
diacs.remove('DIALYTIKA')
diacs.insert(0, 'DIALYTIKA')
name = 'GREEK {} LETTER {}'.format({ True: 'CAPITAL', False: 'SMALL' }[is_capital], LETTERS[letter].upper() )
if diacs:
name += ' WITH ' + ' AND '.join(diacs)
name = IRREGULAR.get(name, name)
try:
found = unicodedata.lookup(name)
except KeyError:
raise KeyError(u'Not found {} (name: {})'.format(beta, name))
t.add(beta, found)
def katakana2hiragana(self, char):
"""カタカナを平仮名に変換する"""
if not self.is_katakana(char):
raise CharTypeException(char)
name = re.sub(r"^KATAKANA\s", "HIRAGANA ", unicodedata.name(char))
return unicodedata.lookup(name)
def __init__(self):
self.proper_html = re.compile(r'<html[^>]*>[\S\s]+</html>', re.UNICODE|re.IGNORECASE)
self.html_tags = re.compile(r'<[^>]+>')
self.html_scripts = re.compile(r'<script[^>]*>[\S\s]*?</script>', re.UNICODE|re.IGNORECASE)
self.html_style = re.compile(r'<style[^>]*>[\S\s]*?</style>', re.UNICODE|re.IGNORECASE)
self.whitespace_chars = re.compile(r'[\s]+', re.UNICODE) # {2,}')
self.unknown_char_seq = re.compile(r'['+ unicodedata.lookup('REPLACEMENT CHARACTER') +']', re.UNICODE|re.IGNORECASE) # {2,}')
def clean_headword(headword):
headword = clean_head.sub('', headword)
headword = re.sub('“|”', '', headword)
headword = re.sub('[*/_0-9\[\]!?]', '', headword)
headword = re.sub('[.,]', '', headword)
headword = re.sub('[ ][ ]+', ' ', headword)
#headword = re.sub(' \(.*\)', '', headword)
headword = unicode(headword)
newheadword = ''
for char in headword:
charname = ud.name(char)
if 'LATIN' in charname:
newcharname = re.search('LATIN (CAPITAL|SMALL) LETTER [\w]', charname).group(0)
else:
newcharname = charname
newheadword += ud.lookup(newcharname)
'''
headword = headword.lower()
headword = re.sub('Ï|İ|Î|Ī|Ì|Í', 'i', headword)
headword = re.sub('Ç|ç|Ć|Č', 'c', headword)
headword = re.sub('Ş|Š', 's', headword)
headword = re.sub('Ḥ', 'h', headword)
headword = re.sub('Á|Â|Ă|À|Ã|Ā|ă', 'a', headword)
headword = re.sub('É|È|Ë|Ē|Ê', 'e', headword)
headword = re.sub('Ü|Ù|Ú|Ū|Û', 'u', headword)
headword = re.sub('Ő|Ö|Ô|Ó|Ò|ö', 'o', headword)
headword = re.sub('Ż|Ž', 'z', headword)
headword = re.sub('Ġ|Ğ', 'g', headword)
headword = re.sub('Ţ', 't', headword)
headword = re.sub('Ÿ', 'y', headword)
headword = re.sub('Ñ', 'n', headword)
'''
return newheadword.lower()
def normalize_char(c):
try:
cname = unicodedata.name( unicode(c) )
cname = cname[:cname.index( ' WITH' )]
return unicodedata.lookup( cname )
except ( ValueError, KeyError ):
return c
def test_string_encoding(self):
ezs = ezstruct.Struct(
"NET_ENDIAN",
ezstruct.Field("STRING", string_encoding="ascii", name="ascii", length=ezstruct.Delimiter(b"\xFF")),
ezstruct.Field("STRING", string_encoding="utf-8", name="utf-8", length=ezstruct.Delimiter(b"\xFF")),
ezstruct.Field("STRING", string_encoding="utf-16-le", name="utf-16-le", length=ezstruct.Delimiter(b"\xFF")),
)
self.roundTrip(ezs, b"cat\xFFdog\xFFf\x00o\x00x\x00\xFF", {"ascii": "cat", "utf-8": "dog", "utf-16-le": "fox"})
self.roundTrip(
ezs,
b"meow\xFFw\xC3\xB6\xC3\xB6f\xFF\x34\xD8\x1E\xDD\xFF",
{"ascii": "meow", "utf-8": u"w\u00F6\u00F6f", "utf-16-le": unicodedata.lookup("MUSICAL SYMBOL G CLEF")},
)
ezs = ezstruct.Struct(
"NET_ENDIAN",
ezstruct.Field("STRING", string_encoding="ascii", length=1, name="a"),
ezstruct.Field(
"STRING", string_encoding="ascii", length=1, name="b", string_encoding_errors_policy="backslashreplace"
),
)
self.assertRaisesRegex(
ValueError,
(r"""'ascii' codec can't encode character u?'\\xf6' """ r"""in position 0: ordinal not in range\(128\)"""),
ezs.pack_bytes,
{"a": u"\u00F6", "b": "x"},
)
self.assertEqual(b"x\\xf6", ezs.pack_bytes({"a": "x", "b": u"\u00F6"}))
def simplify_el(string):
result = []
for c in string:
try:
name = unicodedata.name(c).split()
except ValueError:
continue
if 'WITH' in name:
assert name[4] == 'WITH'
# possible diacritics: TONOS OXIA DIALYTIKA VARIA DASIA
# PERISPOMENI PROSGEGRAMMENI YPOGEGRAMMENI
diacritics = []
if 'DIALYTIKA' in name[5:]:
diacritics.append('DIALYTIKA')
if any(a in name[5:]
for a in ['TONOS', 'OXIA', 'VARIA', 'PERISPOMENI']):
diacritics.append('TONOS')
new_name = name[:4]
if len(diacritics) >= 1:
new_name += ['WITH', diacritics[0]]
for d in diacritics[1:]:
new_name += ['AND', d]
result.append(unicodedata.lookup(' '.join(new_name)))
else:
# oxia, tonos, acute, RIGHT SINGLE QUOTATION MARK (they look the same)
if c in ('´', '΄', '´', '’'):
c = "'"
result.append(c)
result = ''.join(result)
result = ' '.join(convert_numbers(word)
for word in result.split())
return result
def setup(app):
app.add_builder(builder.SlideBuilder)
app.add_builder(builder.DirectorySlideBuilder)
app.connect('html-page-context', slides.slide_context)
app.connect('html-collect-pages', slides.get_pages)
# core slide configuration
app.add_config_value('slide_theme', 'slides', 'html')
app.add_config_value('slide_levels', 3, 'html')
app.add_config_value('slide_theme_options', {}, 'html')
app.add_config_value('slide_theme_path', [], 'html')
app.add_config_value('slide_numbers', False, 'html')
# support for linking html output to slides
app.add_config_value('slide_link_html_to_slides', False, 'html')
app.add_config_value('slide_link_html_sections_to_slides', False, 'html')
app.add_config_value('slide_relative_path', '../slides/', 'html')
app.add_config_value('slide_html_slide_link_symbol',
unicodedata.lookup('section sign'), 'html')
# support for linking from slide output to html
app.add_config_value('slide_link_to_html', False, 'html')
app.add_config_value('slide_html_relative_path', '../html/', 'html')
# slide-related directives
app.add_node(directives.slides)
app.add_directive('notslides', directives.Slides)
app.add_directive('slides', directives.Slides)
app.connect('doctree-resolved', directives.process_slide_nodes)
app.add_directive('slideconf', directives.SlideConf)
app.connect('doctree-resolved', directives.process_slideconf_nodes)
app.connect('builder-inited', html.inspect_config)
app.connect('html-page-context', html.add_link)
def half2full(self, char):
u"""半角カタカナ char を全角カタカナに変換する"""
if not self.is_halfwidthkatakana(char):
raise CharTypeException(char)
name = re.sub(r"^HALFWIDTH\s", "", unicodedata.name(char))
return unicodedata.lookup(name)
def hiragana2katakana(self, char):
"""ひらがなをカタカナに変換する"""
if not self.is_hiragana(char):
raise CharTypeException(char)
name = re.sub(r"^HIRAGANA\s", "KATAKANA ", unicodedata.name(char))
return unicodedata.lookup(name)
class CheckersBoard(object):
""" Represents a Checkers Board
A Checkers board is a matrix, laid out as follows:
A B C D E F G H
0 * - * - * - * -
1 - * - * - * - *
2 * - * - * - * -
3 - * - * - * - *
4 * - * - * - * -
5 - * - * - * - *
6 * - * - * - * -
7 - * - * - * - *
Dashes (-) are white spaces
Asterics (*) are black spaces
Checkers boards should be immutable to prevent accidental modification
#"LARGE BLUE CIRCLE"
"""
board_symbol_mapping = {
0: u' ',
1: unicodedata.lookup("LARGE RED CIRCLE"), #normal 'white' piece
2: unicodedata.lookup("MEDIUM WHITE CIRCLE"), #normal 'black' piece
3: unicodedata.lookup("HEAVY LARGE CIRCLE"), #king 'white' piece
4:
unicodedata.lookup("WHITE MEDIUM SMALL SQUARE"), #king 'black' piece
5: unicodedata.lookup("BLACK LARGE SQUARE")
}
def __init__(self,
_boardArray=None,
_currentPlayer=1,
_drawCounter=40,
_moveCounter=0):
if _boardArray is None:
# Normal Starting Board
self.boardArray = (
(5, 2, 5, 2, 5, 2, 5, 2),
(2, 5, 2, 5, 2, 5, 2, 5),
(5, 2, 5, 2, 5, 2, 5, 2),
(0, 5, 0, 5, 0, 5, 0, 5),
(5, 0, 5, 0, 5, 0, 5, 0),
(1, 5, 1, 5, 1, 5, 1, 5),
(5, 1, 5, 1, 5, 1, 5, 1),
(1, 5, 1, 5, 1, 5, 1, 5),
)
## TEST BOARDS ##
# 1 piece each, opposite ends
# self.boardArray = ( ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 1, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 2, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ), )
# King, Five jumps
# self.boardArray = ( ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 2, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 3, 5, 0, 5, 0 ),
# ( 0, 5, 2, 5, 2, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 2, 5, 2, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ), )
# Player 2, close win
# self.boardArray = ( ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 2, 5, 0, 5, 2 ),
# ( 2, 5, 0, 5, 2, 5, 1, 5 ),
# ( 5, 2, 5, 0, 5, 1, 5, 2 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 4, 5, 0, 5, 0, 5, 0, 5 ), )
# Possible draw
# self.boardArray = ( ( 5, 3, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 1, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 2, 5, 2, 5 ),
# ( 5, 2, 5, 4, 5, 0, 5, 1 ),
# ( 4, 5, 4, 5, 0, 5, 4, 5 ), )
# White victory through no opponent moves
# self.boardArray = ( ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 2, 5, 2, 5, 2, 5, 2, 5 ),
# ( 5, 2, 5, 2, 5, 2, 5, 2 ),
# ( 1, 5, 1, 5, 1, 5, 1, 5 ),
# ( 5, 1, 5, 0, 5, 0, 5, 1 ),
# ( 0, 5, 4, 5, 2, 5, 0, 5 ),
# ( 5, 1, 5, 1, 5, 2, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ), )
# Tricky finish
# self.boardArray = ( ( 5, 3, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 2, 5, 4, 5, 0, 5, 2, 5 ),
# ( 5, 0, 5, 0, 5, 0, 5, 0 ),
# ( 0, 5, 0, 5, 0, 5, 0, 5 ),
# ( 5, 2, 5, 0, 5, 0, 5, 2 ),
# ( 4, 5, 0, 5, 0, 5, 0, 5 ), )
else:
# store an immutable copy
self.boardArray = tuple(map(tuple, _boardArray))
self.currentPlayer = _currentPlayer
self.boardWidth = 8
self.boardHeight = 8
self.drawCounter = _drawCounter # After this many moves, consider the game a draw
self.moveCounter = _moveCounter
def getCurrentPlayerId(self):
return self.currentPlayer
def getCurrentPlayerPieceIds(self):
if self.getCurrentPlayerId() == 1:
return [1, 3]
else:
return [2, 4]
def getOtherPlayerId(self):
if self.getCurrentPlayerId() == 1:
return 2
else:
return 1
def getOtherPlayerPieceIds(self):
if self.getOtherPlayerId() == 1:
return [1, 3]
else:
return [2, 4]
def getBoardArray(self):
return self.boardArray
def getCell(self, row, col):
# return player id in the cell
# if empty return 0 or 5
return self.boardArray[row][col]
def getDrawCounter(self):
return self.drawCounter
def resetDrawCounter(self, _drawCounter=40):
self.drawCounter = _drawCounter
def getMoveCounter(self):
return self.moveCounter
def kingMe(self, board, verbose=False):
for col in range(self.boardWidth):
# Player 1, non-king piece
if board[0][col] == 1:
board[0][col] = 3 # switch to king
# Player 2, non-king piece
if board[7][col] == 2:
board[7][col] = 4 # switch to king
return board
def _getPointFromToken(self, token):
# Interprets tokens as: (column, Row)
# Returns traditional: (row, column)
colstr = "ABCDEFGH"
row = int(token[1])
col = colstr.find(token[0].upper())
return (row, col)
def _getTokenFromPoint(self, point):
# Returns a token on the board from a (row, col) point
colstr = "ABCDEFGH"
row, col = point
if row < 0 or row >= self.boardHeight:
return "-1-1"
if col < 0 or col >= self.boardWidth:
return "-1-1"
token = colstr[col]
token += str(row)
return token
def _isJump(self, piece, move):
prow, pcol = self._getPointFromToken(piece)
mrow, mcol = self._getPointFromToken(move)
rdelta = abs(mrow - prow)
cdelta = abs(mcol - pcol)
if rdelta <= 1 and cdelta <= 1:
return None
# Calculate position of jumped piece
if mrow > prow: # Downward jump
jumpPieceRow = prow + 1
else: # Upward jump
jumpPieceRow = prow - 1
if mcol > pcol: # Right jump
jumpPieceCol = pcol + 1
else: # Left jump
jumpPieceCol = pcol - 1
return (jumpPieceRow, jumpPieceCol)
def moveIsValid(self, piece, move, curBoard):
if len(piece) != 2 or len(move) != 2:
if VERY_VERBOSE:
print("Rule Violation: 0")
return False
prow, pcol = self._getPointFromToken(piece)
mrow, mcol = self._getPointFromToken(move)
# Outside of the board checks
if prow < 0 or prow >= self.boardHeight:
if VERY_VERBOSE:
print("Rule Violation: 1")
return False
if pcol < 0 or pcol >= self.boardWidth:
if VERY_VERBOSE:
print("Rule Violation: 2")
return False
if mrow < 0 or mrow >= self.boardHeight:
if VERY_VERBOSE:
print("Rule Violation: 3")
return False
if mcol < 0 or mcol >= self.boardWidth:
if VERY_VERBOSE:
print("Rule Violation: 4")
return False
# Player slected a valid piece
if curBoard[prow][pcol] not in self.getCurrentPlayerPieceIds():
if VERY_VERBOSE:
print("Rule Violation: 5")
return False
# Move point is not empty
if curBoard[mrow][mcol] != 0:
if VERY_VERBOSE:
print("Rule Violation: 6")
return False
# Player 1 non-King pieces can only move down rows
if curBoard[prow][pcol] == 1:
if mrow >= prow:
if VERY_VERBOSE:
print("Rule Violation: 7")
return False
# Player 2 non-King pieces can only move up rows
if curBoard[prow][pcol] == 2:
if mrow <= prow:
if VERY_VERBOSE:
print("Rule Violation: 8")
return False
# Player must jump over an opponent piece
if self._isJump(piece, move) != None:
jrow, jcol = self._isJump(piece, move)
if curBoard[jrow][jcol] not in self.getOtherPlayerPieceIds():
if VERY_VERBOSE:
print("Rule Violation: 9")
return False
return True
def doMove(self, piece, moveset):
# Execute the specified move as the specified player.
# Return a new board with the result.
if len(moveset) == 0:
raise InvalidMoveException(piece, moveset, self)
if type(moveset) is not list:
moveset = moveset.strip().split()
newBoard = list(map(list, self.getBoardArray()))
for move in moveset:
# Ensure multi-moves are jumps
if len(moveset) > 1:
if not self._isJump(piece, move):
raise InvalidMoveException(piece, move, self)
# Series of checks to ensure the move is valid
if not self.moveIsValid(piece, move, newBoard):
raise InvalidMoveException(piece, move, self)
prow, pcol = self._getPointFromToken(piece)
mrow, mcol = self._getPointFromToken(move)
pieceId = newBoard[prow][pcol]
newBoard[prow][pcol] = 0
newBoard[mrow][mcol] = pieceId
# Performs a piece capture (jump)
if self._isJump(piece, move):
jrow, jcol = self._isJump(piece, move)
newBoard[jrow][jcol] = 0
self.resetDrawCounter()
piece = self._getTokenFromPoint((mrow, mcol))
#Check for new Kings
newBoard = self.kingMe(newBoard)
# Make the board immutable again
newBoard = tuple(map(tuple, newBoard))
return CheckersBoard(newBoard, self.getOtherPlayerId(),
self.drawCounter - 1, self.moveCounter + 1)
def isWin(self):
whitePieces, blackPieces = self.getPieceCount()
if whitePieces == 0:
return 2
if blackPieces == 0:
return 1
return 0
def isDraw(self):
if self.drawCounter == 0:
return True
return False
def isGameOver(self):
if self.isWin() != 0:
return True
if self.isDraw():
return True
return False
def copy(self):
return CheckersBoard(self.getBoardArray(), self.getCurrentPlayerId(),
self.drawCounter, self.moveCounter)
def getPieceConfig(self):
plist = [0, 0, 0, 0] # red, white, red king, white king
for i in range(self.boardHeight):
for j in range(self.boardWidth):
pid = self.getCell(i, j)
if pid == 0 or pid == 5:
continue
plist[pid - 1] += 1
return plist
def getPieceCount(self):
red = 0
white = 0
for i in range(self.boardHeight):
for j in range(self.boardWidth):
pid = self.getCell(i, j)
if pid in [1, 3]:
red += 1
elif pid in [2, 4]:
white += 1
return (red, white)
def getPieceLocations(self, reqToken=False):
pdict = {1: [], 2: []}
for i in range(self.boardHeight):
for j in range(self.boardWidth):
pid = self.getCell(i, j)
if pid not in [0, 5]:
loc = (i, j)
if reqToken:
loc = _getTokenFromPoint(loc)
else:
continue
if pid in [1, 3]:
pdict[1].append(loc)
elif pid in [2, 4]:
pdict[2].append(loc)
return pdict
def getAllPlayerPieces(self, playerid):
pieceSet = []
pid = 0
if playerid == 1:
pid = (1, 3)
else:
pid = (2, 4)
for i in range(self.boardHeight):
for j in range(self.boardWidth):
if self.getCell(i, j) in pid:
pieceSet.append((i, j))
return pieceSet
def __str__(self):
""" Return a string representation of this board """
retVal = [" " + ' '.join([str(x) for x in "ABCDEFGH"])]
retVal += [
str(i) + ' ' +
' '.join([self.board_symbol_mapping[x] for x in row])
for i, row in enumerate(self.boardArray)
]
return '\n' + '\n'.join(retVal) + '\n'
def __repr__(self):
""" The string representation of a board in the Python shell """
return self.__str__()
def __hash__(self):
""" Determine the hash key of a board. The hash key must be the same on any two identical boards. """
return self.boardArray.__hash__()
def __eq__(self, other):
""" Determine whether two boards are equal. """
return (self.getBoardArray() == other.getBoardArray())
-This module provides access to UCD and uses the same symbols
-and names as defined by the Unicode Character Database.
## Function
1-unicodedata.lookup(name)
2-unicodedata.name(chr[, default])
3-unicodedata.decimal(chr[, default])
4-unicodedata.digit(chr[, default])
5-unicodedata.numeric(chr[, default])
6-unicodedata.category(chr)
7-unicodedata.bidirectional(chr)
8-unicodedata.normalize(form, unistr)
'''
import unicodedata
print(unicodedata.lookup('LEFT CURLY BRACKET'))
print(unicodedata.lookup('RIGHT CURLY BRACKET'))
print(unicodedata.lookup('ASTERISK'))
#############################
print(unicodedata.name(u'/'))
print(unicodedata.name(u'|'))
print(unicodedata.name(u':'))
################################
print(unicodedata.decimal(u'9'))
print(unicodedata.decimal(u'5'))
####################################
print(unicodedata.decimal(u'0'))
print(unicodedata.decimal(u'1'))
###################################
print(unicodedata.category(u'A'))
print(unicodedata.category(u'b'))
def test_bmp_characters(self):
for code in range(0x10000):
char = chr(code)
name = unicodedata.name(char, None)
if name is not None:
self.assertEqual(unicodedata.lookup(name), char)
def _escape(source, escape, state):
# handle escape code in expression
code = CATEGORIES.get(escape)
if code:
return code
code = ESCAPES.get(escape)
if code:
return code
try:
c = escape[1:2]
if c == "x":
# hexadecimal escape
escape += source.getwhile(2, HEXDIGITS)
if len(escape) != 4:
raise source.error("incomplete escape %s" % escape,
len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "u" and source.istext:
# unicode escape (exactly four digits)
escape += source.getwhile(4, HEXDIGITS)
if len(escape) != 6:
raise source.error("incomplete escape %s" % escape,
len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "U" and source.istext:
# unicode escape (exactly eight digits)
escape += source.getwhile(8, HEXDIGITS)
if len(escape) != 10:
raise source.error("incomplete escape %s" % escape,
len(escape))
c = int(escape[2:], 16)
chr(c) # raise ValueError for invalid code
return LITERAL, c
elif c == "N" and source.istext:
import unicodedata
# named unicode escape e.g. \N{EM DASH}
if not source.match('{'):
raise source.error("missing {")
charname = source.getuntil('}', 'character name')
try:
c = ord(unicodedata.lookup(charname))
except KeyError:
raise source.error("undefined character name %r" % charname,
len(charname) + len(r'\N{}'))
return LITERAL, c
elif c == "0":
# octal escape
escape += source.getwhile(2, OCTDIGITS)
return LITERAL, int(escape[1:], 8)
elif c in DIGITS:
# octal escape *or* decimal group reference (sigh)
if source.next in DIGITS:
escape += source.get()
if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS
and source.next in OCTDIGITS):
# got three octal digits; this is an octal escape
escape += source.get()
c = int(escape[1:], 8)
if c > 0o377:
raise source.error(
'octal escape value %s outside of '
'range 0-0o377' % escape, len(escape))
return LITERAL, c
# not an octal escape, so this is a group reference
group = int(escape[1:])
if group < state.groups:
if not state.checkgroup(group):
raise source.error("cannot refer to an open group",
len(escape))
state.checklookbehindgroup(group, source)
return GROUPREF, group
raise source.error("invalid group reference %d" % group,
len(escape) - 1)
if len(escape) == 2:
if c in ASCIILETTERS:
raise source.error("bad escape %s" % escape, len(escape))
return LITERAL, ord(escape[1])
except ValueError:
pass
raise source.error("bad escape %s" % escape, len(escape))
def _class_escape(source, escape):
# handle escape code inside character class
code = ESCAPES.get(escape)
if code:
return code
code = CATEGORIES.get(escape)
if code and code[0] is IN:
return code
try:
c = escape[1:2]
if c == "x":
# hexadecimal escape (exactly two digits)
escape += source.getwhile(2, HEXDIGITS)
if len(escape) != 4:
raise source.error("incomplete escape %s" % escape,
len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "u" and source.istext:
# unicode escape (exactly four digits)
escape += source.getwhile(4, HEXDIGITS)
if len(escape) != 6:
raise source.error("incomplete escape %s" % escape,
len(escape))
return LITERAL, int(escape[2:], 16)
elif c == "U" and source.istext:
# unicode escape (exactly eight digits)
escape += source.getwhile(8, HEXDIGITS)
if len(escape) != 10:
raise source.error("incomplete escape %s" % escape,
len(escape))
c = int(escape[2:], 16)
chr(c) # raise ValueError for invalid code
return LITERAL, c
elif c == "N" and source.istext:
import unicodedata
# named unicode escape e.g. \N{EM DASH}
if not source.match('{'):
raise source.error("missing {")
charname = source.getuntil('}', 'character name')
try:
c = ord(unicodedata.lookup(charname))
except KeyError:
raise source.error("undefined character name %r" % charname,
len(charname) + len(r'\N{}'))
return LITERAL, c
elif c in OCTDIGITS:
# octal escape (up to three digits)
escape += source.getwhile(2, OCTDIGITS)
c = int(escape[1:], 8)
if c > 0o377:
raise source.error(
'octal escape value %s outside of '
'range 0-0o377' % escape, len(escape))
return LITERAL, c
elif c in DIGITS:
raise ValueError
if len(escape) == 2:
if c in ASCIILETTERS:
raise source.error('bad escape %s' % escape, len(escape))
return LITERAL, ord(escape[1])
except ValueError:
pass
raise source.error("bad escape %s" % escape, len(escape))
import logging
import os
import unicodedata
import appdirs
import numpy as np
units = {
"EURO": u"{}".format(unicodedata.lookup("EURO SIGN")),
"DOLLARS": "$",
'LPS': ['m', 'mm'],
'LPM': ['m', 'mm'],
'MLD': ['m', 'mm'],
'CMH': ['m', 'mm'],
'CMD': ['m', 'mm'],
# now imperial units
'CFS': ['ft', 'in'],
'GPM': ['ft', 'in'],
'MGD': ['ft', 'in'],
'IMGD': ['ft', 'in'],
'AFD': ['ft', 'in'],
}
curve_colors = ["b", "r", "p", "g"]
widget_select_background = 19
widget_default_background = 10 # see http://doc.qt.io/qt-4.8/qpalette.html#ColorRole-enum
_appdir = appdirs.user_data_dir("RRPAMWDS", "ASSELAPAT")
# PROJECTDATA = os.path.join(_appdir, "rrpamwds.dat")
# USERDATA = appdirs.user_data_dir(None, None)
PROJECTEXTENSION = ".rrp"
HOMEDIR = os.path.expanduser("~")
PROJECTDATADIREXT = ".datadir"
import math, string, unicodedata
from ..command import *
from discord.utils import get
abc_emoji = {
letter: unicodedata.lookup('REGIONAL INDICATOR SYMBOL LETTER %s' % letter)
for letter in string.ascii_uppercase
}
@command
async def reactlevel(ctx, message_id, channel=None):
if channel is None:
channel = ctx.channel
elif isinstance(channel, int):
channel = get(ctx.bot.get_all_channels(), id=channel)
elif isinstance(channel, str):
channel = get(ctx.guild.text_channels, name=channel) or get(
ctx.bot.get_all_channels(), name=channel)
async for msg in channel.history(limit=100):
if msg.id == message_id:
break
else:
return 'Message not found'
if not msg.reactions:
return 'No reactions'
target = math.ceil(
sum([r.count for r in msg.reactions]) / len(msg.reactions))
('LATIN SMALL LETTER AE', 1),
('SPACE', 1),
('NO-BREAK SPACE', 1),
('CJK COMPATIBILITY IDEOGRAPH-F920', 2),
('MALAYALAM VOWEL SIGN UU', 0),
('ZERO WIDTH SPACE', 0),
('ZERO WIDTH NO-BREAK SPACE', 0),
('COMBINING PALATALIZED HOOK BELOW', 0),
('COMBINING GRAVE ACCENT', 0),
)
nonprinting = '\r\n\t\a\b\f\v\x7f'
import unicodedata
for name, printwidth in samples:
uchr = unicodedata.lookup(name)
calculatedwidth = wcwidth(uchr)
assert calculatedwidth == printwidth, \
'width for %r should be %d, but is %d?' % (uchr, printwidth, calculatedwidth)
for c in nonprinting:
calculatedwidth = wcwidth(c)
assert calculatedwidth < 0, \
'%r is a control character, but wcwidth gives %d' % (c, calculatedwidth)
assert wcwidth('\0') == 0 # special case
# depending on how python is compiled, code points above U+FFFF may not be
# treated as single characters, so ord() won't work. test a few of these
# manually.
def _replace_escape_unicode(self, range, value):
chunks = []
offset = 0
while offset < len(value):
match = self._lex_escape_unicode_re.search(value, offset)
if match is None:
# Append the remaining of the string
chunks.append(value[offset:])
break
# Append the part of string before match
chunks.append(value[offset:match.start()])
offset = match.end()
# Process the escape
if match.group(1) is not None: # single-char
chr = match.group(1)
if chr == "\n":
pass
elif chr == "\\" or chr == "'" or chr == "\"":
chunks.append(chr)
elif chr == "a":
chunks.append("\a")
elif chr == "b":
chunks.append("\b")
elif chr == "f":
chunks.append("\f")
elif chr == "n":
chunks.append("\n")
elif chr == "r":
chunks.append("\r")
elif chr == "t":
chunks.append("\t")
elif chr == "v":
chunks.append("\v")
elif match.group(2) is not None: # oct
chunks.append(unichr(int(match.group(2), 8)))
elif match.group(3) is not None: # hex
chunks.append(unichr(int(match.group(3), 16)))
elif match.group(4) is not None: # unicode-16
chunks.append(unichr(int(match.group(4), 16)))
elif match.group(5) is not None: # unicode-32
try:
chunks.append(unichr(int(match.group(5), 16)))
except ValueError:
error = diagnostic.Diagnostic(
"error", "unicode character out of range", {},
source.Range(self.source_buffer,
range.begin_pos + match.start(0),
range.begin_pos + match.end(0)))
self.diagnostic_engine.process(error)
elif match.group(6) is not None: # unicode-name
try:
chunks.append(unicodedata.lookup(match.group(6)))
except KeyError:
error = diagnostic.Diagnostic(
"error", "unknown unicode character name", {},
source.Range(self.source_buffer,
range.begin_pos + match.start(0),
range.begin_pos + match.end(0)))
self.diagnostic_engine.process(error)
return "".join(chunks)
if re.match(r'^[a-fA-F0-9]+$', s):
return self.self_insert(key=chr(int(s, base=16)))
else:
return self.self_insert(key=unicodedata.lookup(s))
# wherein we work out and write down a bunch of compose key sequences for
# various interesting non-ASCII things
COMPOSE_SEQUENCES = []
# just see what canonical compositions we can find in unicode and bind them
for key, diacritic in [('`', 'grave accent'), ("'", 'acute accent'),
("^", 'circumflex accent'), ('"', 'diaeresis'),
(',', 'cedilla'), ('=', 'macron'), ('~', 'tilde')]:
combiner = unicodedata.lookup('combining ' + diacritic)
COMPOSE_SEQUENCES.append((key + ' [space]', combiner))
COMPOSE_SEQUENCES.append((key + ' ' + key, unicodedata.lookup(diacritic)))
for letter in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ':
char = unicodedata.normalize('NFC', letter + combiner)
if len(char) == 1:
COMPOSE_SEQUENCES.append((key + ' ' + letter, char))
COMPOSE_SEQUENCES += [
('1 / 4', '\N{vulgar fraction one quarter}'),
('1 / 2', '\N{vulgar fraction one half}'),
('3 / 4', '\N{vulgar fraction three quarters}'),
('1 / 7', '\N{vulgar fraction one seventh}'),
('1 / 9', '\N{vulgar fraction one ninth}'),
('1 / 1 0', '\N{vulgar fraction one tenth}'),
('1 / 3', '\N{vulgar fraction one third}'),
"EN SPACE",
"EM SPACE",
"THREE-PER-EM SPACE",
"FOUR-PER-EM SPACE",
"SIX-PER-EM SPACE",
"FIGURE SPACE",
"PUNCTUATION SPACE",
"THIN SPACE",
"HAIR SPACE",
"NARROW NO-BREAK SPACE",
"MEDIUM MATHEMATICAL SPACE",
"IDEOGRAPHIC SPACE",
]
typographic_whitespace = "".join(
{unicodedata.lookup(n)
for n in typographic_whitespace_names})
def normalize_overall_whitespace(html,
keep_typographic_whitespace=False,
whitespace_re=None):
# remove all sorts of newline and nbsp characters
whitespace = ["\n", " ", "
", "\r", " ", "
"]
if not keep_typographic_whitespace:
# non-breaking space representations
whitespace += ["\xa0", " ", " ", " "]
if whitespace_re is None:
whitespace_re = r"\s+"
for ch in whitespace:
html = html.replace(ch, " ")
import re
import pathlib
import unicodedata
from urllib.parse import quote
from bs4 import BeautifulSoup
# pre-define the list of HTML files we want to miodify
FILES_TO_MODIFY = [
"advanced_usage.html", "api.html", "builtin_models.html",
"automated_configuration.html", "contributing.html",
"custom_notebooks.html", "evaluation.html", "getting_started.html",
"pipeline.html", "internal.html", "tutorial.html", "usage_rsmtool.html",
"utilities.html", "who.html"
]
PILCROW = unicodedata.lookup('PILCROW SIGN')
def main(): # noqa: D103
# set up an argument parser
parser = argparse.ArgumentParser(prog='add_dash_anchors.py')
parser.add_argument("htmldir",
type=pathlib.Path,
help="path to the already-built HTML documentation")
# parse given command line arguments
args = parser.parse_args()
# set up the logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
def unicode_test(value):
import unicodedata
name = unicodedata.name(value)
value2 = unicodedata.lookup(name)
print('value="%s", name="%s", value2="%s"' % (value, name, value2))
def set_results(self, results):
"""Set the input results."""
# false positive, pylint: disable=no-member
prev_sel_learner = self.selected_learner.copy()
self.clear()
self.warning()
self.closeContext()
data = None
if results is not None and results.data is not None:
data = results.data[results.row_indices]
self.Error.no_regression.clear()
self.Error.empty_input.clear()
if data is not None and not data.domain.has_discrete_class:
self.Error.no_regression()
data = results = None
elif results is not None and not results.actual.size:
self.Error.empty_input()
data = results = None
nan_values = False
if results is not None:
assert isinstance(results, Orange.evaluation.Results)
if np.any(np.isnan(results.actual)) or \
np.any(np.isnan(results.predicted)):
# Error out here (could filter them out with a warning
# instead).
nan_values = True
results = data = None
self.Error.invalid_values(shown=nan_values)
self.results = results
self.data = data
if data is not None:
class_values = data.domain.class_var.values
elif results is not None:
raise NotImplementedError
if results is None:
self.report_button.setDisabled(True)
return
self.report_button.setDisabled(False)
nmodels = results.predicted.shape[0]
self.headers = class_values + \
(unicodedata.lookup("N-ARY SUMMATION"), )
# NOTE: The 'learner_names' is set in 'Test Learners' widget.
self.learners = getattr(results, "learner_names",
[f"Learner #{i + 1}" for i in range(nmodels)])
self._init_table(len(class_values))
self.openContext(data.domain.class_var)
if not prev_sel_learner or prev_sel_learner[0] >= len(self.learners):
if self.learners:
self.selected_learner[:] = [0]
else:
self.selected_learner[:] = prev_sel_learner
self._update()
self._set_selection()
self.commit.now()
'w': 'GREEK SMALL LETTER OMEGA', # ω ω
final_s: 'GREEK SMALL LETTER FINAL SIGMA', # ς ς
}
raw_accents = {
">": 'COMBINING GRAVE ACCENT', # 'U+0300', 768,
"<": 'COMBINING ACUTE ACCENT', # 'U+0301', 769,
"^": 'COMBINING INVERTED BREVE', # 'U+0311', 785,
"'": 'COMBINING COMMA ABOVE', # 'U+0313', 787, smooth breathing
"`": 'COMBINING REVERSED COMMA ABOVE', # 'U+0314', 788, rough breathing
}
accent_entities = []
for raw_char, unicode_accent_name in raw_accents.items(): #{
unicode_char = unicodedata.lookup( unicode_accent_name )
entity_code = '&#%d;' % ord( unicode_char )
accent_entities.append( entity_code )
#}
numeric_non_break_space = ' ' # will use this instead of as it fits in later with the
# formatting done in writeHTML.py
greek_start = '$sp$$'
greek_end = '$rm' #the full code is '$rm$$', but some entries are missing one or both final dollar signs
full_greek_end = '$rm$$'
unidentified_chars = []
percent = '%'
newline = '\n'
space = ' '
def changeOneTextField( handle, table_name, field_name ): #{
global unidentified_chars
the_database_connection = c.get_database_connection()
the_cursor = the_database_connection.cursor()
statement = "select distinct entry_id, %s from %s " % (field_name, table_name)
statement += " where %s like '%s%s%s'" % (field_name, percent, greek_start, percent)
statement += " order by entry_id, %s" % field_name
#print statement
the_cursor.execute( statement )
results = the_cursor.fetchall()
for row in results: #{
entry_id = row[ 0 ]
fieldval = row[ 1 ]
field_parts = fieldval.split( greek_start )
index = -1
for section in field_parts: #{
index += 1
if index == 0: continue # before first bit of Greek
sub_sections = section.split( greek_end )
encoded_latin = sub_sections[ 0 ].strip()
orig_encoded_latin = encoded_latin
greek = u''
print entry_id, encoded_latin
# We need to change s/sigma at the end of words to 'final sigma'
final_char = encoded_latin[ -1 : ]
if final_char == 's': #{
encoded_latin = '%s%s' % (encoded_latin[ 0 : -1 ], final_s)
#}
encoded_latin = encoded_latin.replace( 's ', final_s + ' ' )
print entry_id, encoded_latin
# Now look up the name of the Greek character corresponding to this Latin character
for one_char in encoded_latin[ : ]: #{
greek_char_name = ''
greek_char = ''
if one_char.strip() == '': # whitespace character of some kind
greek += one_char
elif letters.has_key( one_char ): #{
greek_char_name = letters[ one_char ]
#print '%s = %s' % (one_char, greek_char_name)
#}
elif raw_accents.has_key( one_char ): #{
greek_char_name = raw_accents[ one_char ]
#print '%s = %s' % (one_char, greek_char_name)
#}
else:
if one_char not in unidentified_chars: unidentified_chars.append( one_char )
if greek_char_name: #{
greek_char = unicodedata.lookup( greek_char_name )
charnum = ord( greek_char )
greek += '&#%d;' % charnum
#}
#}
# Breathings and accents appear BEFORE capital letters, and are written that way in the English,
# e.g. "'Aposhmeiw<seis". However, combining characters always follow the character to which they
# apply. So we need to do some rearrangement. In practice I think we need to add an extra space
# before the start of the word, for the breathings and accents to sit on.
processed_words = []
words = greek.split()
for word in words: #{
for numeric_entity in accent_entities: #{
if word.startswith( numeric_entity ): #{
word = numeric_non_break_space + ' ' + word
#}
#}
processed_words.append( word )
#}
greek = ' '.join( processed_words )
orig_encoded_latin = orig_encoded_latin.replace( "'", "''" ) # escape single quotes for SQL
handle.write( "update %s set %s = replace( %s, '" % (table_name, field_name, field_name))
handle.write( orig_encoded_latin )
handle.write( "', '" )
handle.write( greek )
handle.write( "' ) where entry_id = %d" % entry_id )
handle.write( " and %s like '%s%s%s';" % (field_name, percent, orig_encoded_latin, percent) )
handle.write( newline + newline )
print ' '
#}
#}
# remove the marker for 'Greek starts here'
handle.write( "update %s set %s = replace( %s, '%s', '' );" \
% (table_name, field_name, field_name, greek_start) )
handle.write( newline )
# remove the marker for 'Greek end here',
# but remember that sometimes one or two dollar signs have been missed off the end
tmp_greek_end = full_greek_end
while len( tmp_greek_end ) >= len( greek_end ): #{
handle.write( "update %s set %s = replace( %s, '%s', '' );" \
% (table_name, field_name, field_name, tmp_greek_end) )
handle.write( newline )
tmp_greek_end = tmp_greek_end[ 0 : -1 ] # trim off the last character
#}
the_cursor.close()
the_database_connection.close()
print '----'
print 'Finished processing %s %s' % (table_name, field_name)
print '----'
print ' '
import unicodedata
replacement_character = unicodedata.lookup('REPLACEMENT CHARACTER')
spaces = list(
map(unicodedata.lookup, [
'SPACE',
'NO-BREAK SPACE',
'THIN SPACE',
'IDEOGRAPHIC SPACE',
]))
def is_uppercase(codepoint):
return unicodedata.category(codepoint) == 'Lu'
def to_unicode(value):
import unicodedata
name = unicodedata.name(value)
value2 = unicodedata.lookup(name)
return (name, value2)
def __init__(self, display, verbose=False,
interpretMaskBits=True, mtvOrigin=afwImage.PARENT, fastMaskDisplay=False,
reopenPlot=False, useSexagesimal=False, dpi=None, *args, **kwargs):
"""
Initialise a matplotlib display
@param fastMaskDisplay If True only show the first bitplane that's
set in each pixel
(e.g. if (SATURATED & DETECTED)
ignore DETECTED)
Not really what we want, but a bit faster
@param interpretMaskBits Interpret the mask value under the cursor
@param mtvOrigin Display pixel coordinates with LOCAL origin
(bottom left == 0,0 not XY0)
@param reopenPlot If true, close the plot before opening it.
(useful with e.g. %ipympl)
@param useSexagesimal If True, display coordinates in sexagesimal
E.g. hh:mm:ss.ss (default:False)
May be changed by calling
display.useSexagesimal()
@param dpi Number of dpi (passed to pyplot.figure)
The `frame` argument to `Display` may be a matplotlib figure; this
permits code such as
fig, axes = plt.subplots(1, 2)
disp = afwDisplay.Display(fig)
disp.scale('asinh', 'zscale', Q=0.5)
for axis, exp in zip(axes, exps):
plt.sca(axis) # make axis active
disp.mtv(exp)
"""
if hasattr(display.frame, "number"): # the "display" quacks like a matplotlib figure
figure = display.frame
else:
figure = None
virtualDevice.DisplayImpl.__init__(self, display, verbose)
if reopenPlot:
pyplot.close(display.frame)
if figure is not None:
self._figure = figure
else:
self._figure = pyplot.figure(display.frame, dpi=dpi)
self._figure.clf()
self._display = display
self._maskTransparency = {None: 0.7}
self._interpretMaskBits = interpretMaskBits # interpret mask bits in mtv
self._fastMaskDisplay = fastMaskDisplay
self._useSexagesimal = [useSexagesimal] # use an array so we can modify the value in format_coord
self._mtvOrigin = mtvOrigin
self._mappable_ax = None
self._colorbar_ax = None
self._image_colormap = pyplot.cm.gray
#
self.__alpha = unicodedata.lookup("GREEK SMALL LETTER alpha") # used in cursor display string
self.__delta = unicodedata.lookup("GREEK SMALL LETTER delta") # used in cursor display string
#
# Support self._scale()
#
self._scaleArgs = dict()
self._normalize = None
#
# Support self._erase(), reporting pixel/mask values, and
# zscale/minmax; set in mtv
#
self._i_setImage(None)
#
# Ignore warnings due to BlockingKeyInput
#
if not verbose:
warnings.filterwarnings("ignore", category=matplotlib.cbook.mplDeprecation)
def __init__(self, unicode_symbol_name):
unicode_symbol = unicodedata.lookup(unicode_symbol_name)
super(SymbolByName, self).__init__(0, unicode_symbol)
def __init__(self, unicode_accent_name):
unicode_accent = unicodedata.lookup('COMBINING ' + unicode_accent_name)
super(Combining, self).__init__(1, '{0}' + unicode_accent)
class ConnectFourBoard(object):
""" Store a Connect-Four Board
Connect-Four boards are intended to be immutable; please don't use
Python wizardry to hack/mutate them. (It won't give you an advantage;
it'll just make the tester crash.)
A Connect-Four board is a matrix, laid out as follows:
0 1 2 3 4 5 6 7
0 * * * * * * * *
1 * * * * * * * *
2 * * * * * * * *
3 * * * * * * * *
4 * * * * * * * *
5 * * * * * * * *
6 * * * * * * * *
Board columns fill from the bottom (ie., row 6).
"""
# The horizontal width of the board
board_width = 7
# The vertical height of the board
board_height = 6
# Map of board ID numbers to display characters used to print the board
board_symbol_mapping = {
0: u' ',
1: unicodedata.lookup("WHITE SMILING FACE"),
2: unicodedata.lookup("BLACK SMILING FACE")
}
board_symbol_mapping_ascii = {0: ' ', 1: 'X', 2: 'O'}
def __init__(self,
board_array=None,
board_already_won=None,
modified_column=None,
current_player=1,
previous_move=-1):
""" Create a new ConnectFourBoard
If board_array is specified, it should be an MxN matrix of iterables
(ideally tuples or lists), that will be used to describe the initial
board state. Each cell should be either '0', meaning unoccupied, or
N for some integer N corresponding to a player number.
board_already_won can optionally be set to either None, or to the id#
of the player who has already won the board.
If modified_column is specified, it should be the index of the last column
that had a token dropped into it.
Both board_already_won and modified_column are used as hints to the
'is_win_for_player()' function. It is fine to not specify them, but if they
are specified, they must be correct.
"""
if sys.stdout.encoding and 'UTF' not in sys.stdout.encoding: # If we don't support Unicode
self.board_symbol_mapping = self.board_symbol_mapping_ascii
if board_array == None:
self._board_array = (
(0, ) * self.board_width, ) * self.board_height
else:
# Make sure we're storing tuples, so that they're immutable
self._board_array = tuple(map(tuple, board_array))
#if board_already_won:
# self._is_win = board_already_won
#elif modified_column:
# self._is_win = self._is_win_from_cell(self.get_height_of_column(modified_column), modified_column)
#else:
self._is_win = self.is_win()
self.current_player = current_player
def get_current_player_id(self):
""" Return the id of the player who should be moving now """
return self.current_player
def get_other_player_id(self):
""" Return the id of the opponent of the player who should be moving now """
if self.get_current_player_id() == 1:
return 2
else:
return 1
def get_board_array(self):
""" Return the board array representing this board (as a tuple of tuples) """
return self._board_array
def get_top_elt_in_column(self, column):
"""
Get the id# of the player who put the topmost token in the specified column.
Return 0 if the column is empty.
"""
for row in self._board_array:
if row[column] != 0:
return row[column]
return 0
def get_height_of_column(self, column):
"""
Return the index of the first cell in the specified column that is filled.
Return ConnectFourBoard.board_height if the column is empty.
"""
if column < 0 or column > 6:
return -10
for i in xrange(self.board_height):
if self._board_array[i][column] != 0:
return i - 1
return self.board_height
def get_cell(self, row, col):
"""
Get the id# of the player owning the token in the specified cell.
Return 0 if it is unclaimed.
"""
return self._board_array[row][col]
def do_move(self, column):
"""
Execute the specified move as the specified player.
Return a new board with the result.
Raise 'InvalidMoveException' if the specified move is invalid.
"""
player_id = self.get_current_player_id()
if self.get_height_of_column(column) > self.board_height:
raise InvalidMoveException(column, self)
new_board = list(transpose(self.get_board_array()))
target_col = [x for x in new_board[column] if x != 0]
target_col = [
0 for x in xrange(self.board_height - len(target_col) - 1)
] + [player_id] + target_col
new_board[column] = target_col
new_board = transpose(new_board)
# Re-immutablize the board
new_board = tuple(map(tuple, new_board))
return ConnectFourBoard(new_board,
board_already_won=self.is_win(),
modified_column=column,
current_player=self.get_other_player_id())
def _is_win_from_cell(self, row, col):
""" Determines if there is a winning set of four connected nodes containing the specified cell """
return (self._max_length_from_cell(row, col) >= 4)
def _max_length_from_cell(self, row, col):
""" Return the max-length chain containing this cell """
return max(
self._contig_vector_length(row, col, (1, 1)) +
self._contig_vector_length(row, col, (-1, -1)) + 1,
self._contig_vector_length(row, col, (1, 0)) +
self._contig_vector_length(row, col, (-1, 0)) + 1,
self._contig_vector_length(row, col, (0, 1)) +
self._contig_vector_length(row, col, (0, -1)) + 1,
self._contig_vector_length(row, col, (-1, 1)) +
self._contig_vector_length(row, col, (1, -1)) + 1)
def _contig_vector_length(self, row, col, direction):
"""
Starting in the specified cell and going a step of direction = (row_step, col_step),
count how many consecutive cells are owned by the same player as the starting cell.
"""
count = 0
playerid = self.get_cell(row, col)
while 0 <= row < self.board_height and 0 <= col < self.board_width and playerid == self.get_cell(
row, col):
row += direction[0]
col += direction[1]
count += 1
return count - 1
def longest_chain(self, playerid):
"""
Returns the length of the longest chain of tokens controlled by this player,
0 if the player has no tokens on the board
"""
longest = 0
for i in xrange(self.board_height):
for j in xrange(self.board_width):
if self.get_cell(i, j) == playerid:
longest = max(longest, self._max_length_from_cell(i, j))
return longest
def _contig_vector_cells(self, row, col, direction):
"""
Starting in the specified cell and going a step of direction = (row_step, col_step),
count how many consecutive cells are owned by the same player as the starting cell.
"""
retVal = []
playerid = self.get_cell(row, col)
while 0 <= row < self.board_height and 0 <= col < self.board_width and playerid == self.get_cell(
row, col):
retVal.append((row, col))
row += direction[0]
col += direction[1]
return retVal[1:]
def _chain_sets_from_cell(self, row, col):
""" Return the max-length chain containing this cell """
return [
tuple(x) for x in [
reverse(self._contig_vector_cells(row, col, (1, 1))) +
[(row, col)] + self._contig_vector_cells(row, col, (-1, -1)),
reverse(self._contig_vector_cells(row, col, (1, 0))) +
[(row, col)] + self._contig_vector_cells(row, col, (-1, 0)),
reverse(self._contig_vector_cells(row, col, (0, 1))) +
[(row, col)] + self._contig_vector_cells(row, col, (0, -1)),
reverse(self._contig_vector_cells(row, col, (-1, 1))) +
[(row, col)] + self._contig_vector_cells(row, col, (1, -1))
]
]
def chain_cells(self, playerid):
"""
Returns a set of all cells on the board that are part of a chain controlled
by the specified player.
The return value will be a Python set containing tuples of coordinates.
For example, a return value might look like:
set([ ( (0,1),(0,2),(0,3) ), ( (0,1),(1,1) ) ])
This would indicate a contiguous string of tokens from (0,1)-(0,3) and (0,1)-(1,1).
The coordinates within a tuple are weakly ordered: any coordinates that are
adjacent in a tuple are also adjacent on the board.
Note that single lone tokens are regarded as chains of length 1. This is
sometimes useful, but sometimes not; however, it's relatively easy to remove
such elements via list comprehension or via the built-in Python 'filter' function
as follows (for example):
>>> my_big_chains = filter(lambda x: len(x) > 1, myBoard.chain_cells(playernum))
Also recall that you can convert this set to a list as follows:
>>> my_list = list( myBoard.chain_cells(playernum) )
The return value is provided as a set because sets are unique and unordered,
as is this collection of chains.
"""
retVal = set()
for i in xrange(self.board_height):
for j in xrange(self.board_width):
if self.get_cell(i, j) == playerid:
retVal.update(self._chain_sets_from_cell(i, j))
return retVal
def is_win(self):
"""
Return the id# of the player who has won this game.
Return 0 if it has not yet been won.
"""
#if hasattr(self, "_is_win"):
# return self._is_win
#else:
for i in xrange(self.board_height):
for j in xrange(self.board_width):
cell_player = self.get_cell(i, j)
if cell_player != 0:
win = self._is_win_from_cell(i, j)
if win:
self._is_win = win
return cell_player
return 0
def is_game_over(self):
""" Return True if the game has been won, False otherwise """
return (self.is_win() != 0 or self.is_tie())
def is_tie(self):
""" Return true iff the game has reached a stalemate """
return not 0 in self._board_array[0]
def clone(self):
""" Return a duplicate of this board object """
return ConnectFourBoard(self._board_array,
board_already_won=self._is_win,
current_player=self.get_current_player_id())
def num_tokens_on_board(self):
"""
Returns the total number of tokens (for either player)
currently on the board
"""
tokens = 0
for row in self._board_array:
for col in row:
if col != 0:
tokens += 1
return tokens
def __unicode__(self):
""" Return a string representation of this board """
retVal = [u" " + u' '.join([str(x) for x in range(self.board_width)])]
retVal += [
unicode(i) + ' ' +
u' '.join([self.board_symbol_mapping[x] for x in row])
for i, row in enumerate(self._board_array)
]
return u'\n' + u'\n'.join(retVal) + u'\n'
def __str__(self):
""" Return a string representation of this board """
retVal = [" " + ' '.join([str(x) for x in range(self.board_width)])]
retVal += [
str(i) + ' ' +
' '.join([self.board_symbol_mapping_ascii[x] for x in row])
for i, row in enumerate(self._board_array)
]
return '\n' + '\n'.join(retVal) + '\n'
def __repr__(self):
""" The string representation of a board in the Python shell """
return self.__str__()
def __hash__(self):
""" Determine the hash key of a board. The hash key must be the same on any two identical boards. """
return self._board_array.__hash__()
def __eq__(self, other):
""" Determine whether two boards are equal. """
return (self.get_board_array() == other.get_board_array())
FRACTION_LINE_BEARING = 48 # left and right bearing
FRACTION_LINE_EXTRA_WIDTH = 96
CODEPOINT_FRACTION_LINE = 0xe000 # a private use area codepoint
CONDENSED_SCALE_X = 0.8 # arbitrary
CONDENSED_WIDE_SCALE_X = 1.25 # arbitrary
ITALIC_ANGLE_DEG = 22.5 # see any drafting literature
DIGIT_NAMES = [
"ZERO", "ONE", "TWO", "THREE", "FOUR", "FIVE", "SIX", "SEVEN", "EIGHT",
"NINE"
]
SUPERSCRIPT_DIGIT_CODEPOINTS = [
ord(unicodedata.lookup("SUPERSCRIPT " +
d)) # "SUPERSCRIPT ZERO" through "SUPERSCRIPT NINE"
for d in DIGIT_NAMES
]
SUBSCRIPT_DIGIT_CODEPOINTS = [
ord(unicodedata.lookup("SUBSCRIPT " +
d)) # "SUBSCRIPT ZERO" through "SUBSCRIPT NINE"
for d in DIGIT_NAMES
]
VULGAR_FRACTIONS = [
{
'codepoint': "VULGAR FRACTION ONE QUARTER",
'numerator': "SUPERSCRIPT ONE",
'denominator': "SUBSCRIPT FOUR"
},
def multiplot(self, fig=None):
"""
Generates a single postage stamp plot over a set of ensemble members.
Contains all the main plotting commands or calls to them
Returns the figure object so the user can add to the plot
if required
"""
if fig is None:
self.fig = plt.figure()
else:
self.fig = fig
fig.subplots_adjust(left=0.075, right=0.95, bottom=0.05, top=0.88,
hspace=0.3, wspace=0.3)
# Identify number of ensemble members defined within the cube
nmembers = len(self.layer_list[0].cube.coord('realization').points)
# Save state of cbar variable for later use
plot_colourbar = self.layer_list[0].cbar
# Add the individual postage stamps as subplots
for ens_layer in self.layer_list:
for mem, layer in enumerate(
ens_layer.layer_slice_over('realization')):
# Do not plot colour bar with each postage stamp
# A single colour bar is added to the plot later on
layer.cbar = False
# Specify position of each postage stamp
# and set the projection if requested
if self.projection is not None:
plt.subplot(4, 5, mem + 3, projection=self.projection)
else:
plt.subplot(4, 5, mem + 3)
self.plot_layer(layer, self.mapping)
# Set an axes extent (assuming extent is given in WGS84)
# Needs to follow plot in order that axes have become geoaxes
if self.extent is not None:
plt.gca().set_extent(self.extent, crs=ccrs.PlateCarree())
if self.mapping is not None:
self.plot_map()
# Add subtitle for each postage stamp
mem_title = layer.cube.coord('ensemble_member').points[0]
plt.title(mem_title, fontsize=10)
# Add source location if it exists
coord_names = [coord.name() for coord in layer.cube.coords()]
if 'source_longitude' in coord_names:
cube = layer.cube
sourcelon = cube.coord('source_longitude').points[0]
sourcelat = cube.coord('source_latitude').points[0]
if self.rsmc:
plt.plot(sourcelon, sourcelat, 'ks',
markersize=6, label='Source Location',
transform=ccrs.PlateCarree())
maxval = np.argmax(cube.data)
xyvals = np.unravel_index(maxval, cube.data.shape)
lon_max = cube.coord('longitude').points[xyvals[0]]
lat_max = cube.coord('latitude').points[xyvals[1]]
if cube.attributes['short_name'] != 'TimeOfArrival':
plt.plot(lon_max, lat_max, 'k*',
transform=ccrs.PlateCarree())
else:
plt.plot(sourcelon, sourcelat, 'r*',
markersize=10, label='Source Location',
transform=ccrs.PlateCarree())
ens_layer = self.layer_list[0]
# Add colour bar, titles, etc.
# Set title and suptitle text
if self.title is not None:
title = self.title
else:
title = ens_layer.cube.attributes['Species'] + ' '
title += ens_layer.cube.attributes['Quantity']
cbar_label = ens_layer.cube.attributes['Quantity']
# Add prefix to title if required
if self.titleprefix is not None:
title = self.titleprefix + title
if self.suptitle is not None:
suptitle = self.suptitle
else:
shortname_text = ens_layer.cube.attributes['short_name']
if 'Time Av or Int' in ens_layer.cube.attributes:
tavint_text = ens_layer.cube.attributes['Time Av or Int']
else:
tavint_text = ens_layer.cube.attributes['Time av/int info']
height_text = height_text_fmt(ens_layer.cube)
t_coord = ens_layer.cube.coord('time')
t1 = t_coord.units.num2date(t_coord.bounds[0][0])
t2 = t_coord.units.num2date(t_coord.bounds[0][1])
t_fmt = '%H:%M %d/%m/%Y'
suptitle = 'NAME ensemble with {} members\n'.format(nmembers)
suptitle += '{}, {}, {}\n'.format(shortname_text, height_text,
tavint_text)
suptitle += 'Valid from {} to {}'.format(t1.strftime(t_fmt),
t2.strftime(t_fmt))
# Make a colourbar if required
if plot_colourbar:
ens_layer.cbar = plot_colourbar
cbar_axes = plt.gcf().add_axes([0.14, 0.80, 0.24, 0.02])
ens_layer.construct_cbar(self.mappable,
position=cbar_axes,
orientation='horizontal',
title=title)
plt.suptitle(suptitle, fontsize=14, linespacing=1.2)
# Add branding
if self.mobrand:
insert_logo()
copyright_sign = unicodedata.lookup('COPYRIGHT SIGN')
copyright_txt = copyright_sign + " Met Office Crown Copyright"
plt.annotate(copyright_txt,
(0, 0), (0, -40),
xycoords='axes fraction',
textcoords='offset points',
va='top')
return self.fig
def test_ascii_letters(self):
for char in "".join(map(chr, range(ord("a"), ord("z")))):
name = "LATIN SMALL LETTER %s" % char.upper()
code = unicodedata.lookup(name)
self.assertEqual(unicodedata.name(code), name)
compop_regex = re.compile('[<>=]')
setup_regex = re.compile(r'%setup\b') # intentionally no whitespace before!
setup_q_regex = re.compile(' -[A-Za-z]*q')
setup_t_regex = re.compile(' -[A-Za-z]*T')
setup_ab_regex = re.compile(' -[A-Za-z]*[ab]')
autosetup_regex = re.compile('^\s*%autosetup(\s.*|$)')
autosetup_n_regex = re.compile(' -[A-Za-z]*N')
autopatch_regex = re.compile('^\s*%autopatch(?:\s|$)')
filelist_regex = re.compile('\s+-f\s+\S+')
pkgname_regex = re.compile('\s+(?:-n\s+)?(\S+)')
tarball_regex = re.compile('\.(?:t(?:ar|[glx]z|bz2?)|zip)\\b', re.IGNORECASE)
UNICODE_NBSP = unicodedata.lookup('NO-BREAK SPACE')
def unversioned(deps):
'''Yield unversioned dependency names from the given list.'''
for dep in deps:
if not dep[1]:
yield dep[0]
def contains_buildroot(line):
'''Check if the given line contains use of rpm buildroot.'''
res = rpm_buildroot_regex.search(line)
if res and \
(not res.group(1) or len(res.group(1)) % 2 == 0) and \
(not res.group(2) or len(res.group(2)) % 2 != 0):
special_char = True
elif char == '}':
brace_level -= 1
if brace_level == 0:
special_char = False
return None # case-less
def split_von_last(words):
"""Split "von Last" name into von and Last parts."""
if len(words) > 1 and is_capitalized(words[0]) is False:
for j, word in enumerate(reversed(words[:-1])):
if is_capitalized(word) not in (True, None):
return words[:-j - 1], words[-j - 1:]
return [], words
def lookahead_iter(iterable):
"""Iterator that also yields the next item along with each item. The next
item is `None` when yielding the last item.
"""
items = iter(iterable)
item = next(items)
for next_item in items:
yield item, next_item
item = next_item
yield item, None
EN_DASH = unicodedata.lookup('EN DASH')
def PyUnicode_DecodeUnicodeEscape(s, size, errors):
if (size == 0):
return u''
p = []
pos = 0
while (pos < size):
## /* Non-escape characters are interpreted as Unicode ordinals */
if (s[pos] != '\\'):
p += unichr(ord(s[pos]))
pos += 1
continue
## /* \ - Escapes */
else:
pos += 1
if pos >= len(s):
errmessage = "\\ at end of string"
unicode_call_errorhandler(errors, "unicodeescape", errmessage,
s, pos - 1, size)
ch = s[pos]
pos += 1
## /* \x escapes */
if ch == '\\': p += u'\\'
elif ch == '\'': p += u'\''
elif ch == '\"': p += u'\"'
elif ch == 'b': p += u'\b'
elif ch == 'f': p += u'\014' #/* FF */
elif ch == 't': p += u'\t'
elif ch == 'n': p += u'\n'
elif ch == 'r': p += u'\r'
elif ch == 'v': p += u'\013' #break; /* VT */
elif ch == 'a': p += u'\007' # break; /* BEL, not classic C */
elif '0' <= ch <= '7':
x = ord(ch) - ord('0')
if pos < size:
ch = s[pos]
if '0' <= ch <= '7':
pos += 1
x = (x << 3) + ord(ch) - ord('0')
if pos < size:
ch = s[pos]
if '0' <= ch <= '7':
pos += 1
x = (x << 3) + ord(ch) - ord('0')
p += unichr(x)
## /* hex escapes */
## /* \xXX */
elif ch == 'x':
digits = 2
message = "truncated \\xXX escape"
x = hexescape(s, pos, digits, message, errors)
p += x[0]
pos = x[1]
# /* \uXXXX */
elif ch == 'u':
digits = 4
message = "truncated \\uXXXX escape"
x = hexescape(s, pos, digits, message, errors)
p += x[0]
pos = x[1]
# /* \UXXXXXXXX */
elif ch == 'U':
digits = 8
message = "truncated \\UXXXXXXXX escape"
x = hexescape(s, pos, digits, message, errors)
p += x[0]
pos = x[1]
## /* \N{name} */
elif ch == 'N':
message = "malformed \\N character escape"
#pos += 1
look = pos
try:
import unicodedata
except ImportError:
message = "\\N escapes not supported (can't load unicodedata module)"
unicode_call_errorhandler(errors, "unicodeescape", message,
s, pos - 1, size)
if look < size and s[look] == '{':
#/* look for the closing brace */
while (look < size and s[look] != '}'):
look += 1
if (look > pos + 1 and look < size and s[look] == '}'):
#/* found a name. look it up in the unicode database */
message = "unknown Unicode character name"
st = s[pos + 1:look]
try:
chr = unicodedata.lookup("%s" % st)
except KeyError, e:
x = unicode_call_errorhandler(
errors, "unicodeescape", message, s, pos - 1,
look + 1)
else:
x = chr, look + 1
p += x[0]
pos = x[1]
else:
x = unicode_call_errorhandler(errors, "unicodeescape",
message, s, pos - 1,
look + 1)
else:
x = unicode_call_errorhandler(errors, "unicodeescape",
message, s, pos - 1,
look + 1)
else:
p += '\\'
p += ch
def _init_ui(self):
namesBox = gui.vBox(self.controlArea, "Names")
hbox = gui.hBox(namesBox, margin=0, spacing=0)
gui.lineEdit(
hbox,
self,
"attr1",
"Variable X: ",
controlWidth=80,
orientation=Qt.Horizontal,
callback=self._attr_name_changed,
)
gui.separator(hbox, 21)
hbox = gui.hBox(namesBox, margin=0, spacing=0)
attr2 = gui.lineEdit(
hbox,
self,
"attr2",
"Variable Y: ",
controlWidth=80,
orientation=Qt.Horizontal,
callback=self._attr_name_changed,
)
gui.separator(hbox)
gui.checkBox(
hbox,
self,
"hasAttr2",
"",
disables=attr2,
labelWidth=0,
callback=self.set_dimensions,
)
gui.separator(namesBox)
gui.widgetLabel(namesBox, "Labels")
self.classValuesView = listView = gui.ListViewWithSizeHint(
preferred_size=(-1, 30))
listView.setModel(self.class_model)
itemmodels.select_row(listView, 0)
namesBox.layout().addWidget(listView)
self.addClassLabel = QAction("+",
self,
toolTip="Add new class label",
triggered=self.add_new_class_label)
self.removeClassLabel = QAction(
unicodedata.lookup("MINUS SIGN"),
self,
toolTip="Remove selected class label",
triggered=self.remove_selected_class_label,
)
actionsWidget = itemmodels.ModelActionsWidget(
[self.addClassLabel, self.removeClassLabel], self)
actionsWidget.layout().addStretch(10)
actionsWidget.layout().setSpacing(1)
namesBox.layout().addWidget(actionsWidget)
tBox = gui.vBox(self.controlArea, "Tools", addSpace=True)
buttonBox = gui.hBox(tBox)
toolsBox = gui.widgetBox(buttonBox, orientation=QGridLayout())
self.toolActions = QActionGroup(self)
self.toolActions.setExclusive(True)
self.toolButtons = []
for i, (name, tooltip, tool, icon) in enumerate(self.TOOLS):
action = QAction(name,
self,
toolTip=tooltip,
checkable=tool.checkable,
icon=QIcon(icon))
action.triggered.connect(partial(self.set_current_tool, tool))
button = QToolButton(
iconSize=QSize(24, 24),
toolButtonStyle=Qt.ToolButtonTextUnderIcon,
sizePolicy=QSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.Fixed),
)
button.setDefaultAction(action)
self.toolButtons.append((button, tool))
toolsBox.layout().addWidget(button, i / 3, i % 3)
self.toolActions.addAction(action)
for column in range(3):
toolsBox.layout().setColumnMinimumWidth(column, 10)
toolsBox.layout().setColumnStretch(column, 1)
undo = self.undo_stack.createUndoAction(self)
redo = self.undo_stack.createRedoAction(self)
undo.setShortcut(QKeySequence.Undo)
redo.setShortcut(QKeySequence.Redo)
self.addActions([undo, redo])
self.undo_stack.indexChanged.connect(lambda _: self.invalidate())
gui.separator(tBox)
indBox = gui.indentedBox(tBox, sep=8)
form = QFormLayout(
formAlignment=Qt.AlignLeft,
labelAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow,
)
indBox.layout().addLayout(form)
slider = gui.hSlider(indBox,
self,
"brushRadius",
minValue=1,
maxValue=100,
createLabel=False)
form.addRow("Radius:", slider)
slider = gui.hSlider(indBox,
self,
"density",
None,
minValue=1,
maxValue=100,
createLabel=False)
form.addRow("Intensity:", slider)
slider = gui.hSlider(
indBox,
self,
"symbol_size",
None,
minValue=1,
maxValue=100,
createLabel=False,
callback=self.set_symbol_size,
)
form.addRow("Symbol:", slider)
self.btResetToInput = gui.button(tBox, self, "Reset to Input Data",
self.reset_to_input)
self.btResetToInput.setDisabled(True)
gui.auto_commit(self.left_side, self, "autocommit", "Send")
# main area GUI
viewbox = PaintViewBox(enableMouse=False)
self.plotview = pg.PlotWidget(background="w", viewBox=viewbox)
self.plotview.sizeHint = lambda: QSize(
200, 100) # Minimum size for 1-d painting
self.plot = self.plotview.getPlotItem()
axis_color = self.palette().color(QPalette.Text)
axis_pen = QPen(axis_color)
tickfont = QFont(self.font())
tickfont.setPixelSize(max(int(tickfont.pixelSize() * 2 // 3), 11))
axis = self.plot.getAxis("bottom")
axis.setLabel(self.attr1)
axis.setPen(axis_pen)
axis.setTickFont(tickfont)
axis = self.plot.getAxis("left")
axis.setLabel(self.attr2)
axis.setPen(axis_pen)
axis.setTickFont(tickfont)
if not self.hasAttr2:
self.plot.hideAxis("left")
self.plot.hideButtons()
self.plot.setXRange(0, 1, padding=0.01)
self.mainArea.layout().addWidget(self.plotview)
# enable brush tool
self.toolActions.actions()[0].setChecked(True)
self.set_current_tool(self.TOOLS[0][2])
self.set_dimensions()
def plot(self, fig=None, figsize=None, field_layers=None):
"""
Contains all the main plotting commands or calls to them
Returns the figure object so the user can add to the plot
if required
:param fig: matplotlib figure object on which to create the plot.
:param figsize: Selected size of figure to create if fig is None.
:param field_layers: field layer with which to create plot.
"""
if fig is None:
self.fig = plt.figure()
else:
self.fig = fig
# Examine first layer to extract some properties of the plot:
if field_layers is not None:
layer_list = [layer for layer in field_layers]
else:
layer_list = [layer for layer in self.layer_list]
layer = layer_list[0]
if layer.plottype in ['pcolormesh', 'contour', 'contourf',
'contourf-edge']:
self.x_coord = layer.cube.dim_coords[-2] # lat-type coord
self.y_coord = layer.cube.dim_coords[-1] # lon-type coord
elif layer.plottype == 'scatter_latlon':
self.x_coord = layer.cube.coord('longitude')
self.y_coord = layer.cube.coord('latitude')
# Amend figsize to fit completed plot unless figsize is set manually:
if figsize in [None, 'Auto', 'Default']:
# Automatically determine figure size if it isn't set by user.
self.auto_figsize()
else:
# If figure size is set by user, just make the figure that size.
resize_fig(self.fig, figsize)
# Make sure to specify fig to draw on:
fig = self.fig
# Set the projection if requested
if self.projection is not None:
if self.annote_location == 'right':
plt.subplot2grid((1, 3), (0, 0), colspan=2,
projection=self.projection)
elif self.annote_location == 'below':
plt.subplot2grid((3, 1), (0, 0), rowspan=2,
projection=self.projection)
else:
plt.axes(projection=self.projection)
elif self.annote_location == 'right':
plt.subplot2grid((1, 3), (0, 0), colspan=2)
elif self.annote_location == 'below':
plt.subplot2grid((3, 1), (0, 0), rowspan=2)
# Find field layer(s) and plot it/them.
for layer in layer_list:
self.plot_layer(layer, self.mapping)
if self.mapping is not None:
self.plot_map()
# Add source location if it exists
coord_names = [coord.name()
for coord in self.layer_list[0].cube.coords()]
if 'source_longitude' in coord_names:
lyr_cube = self.layer_list[0].cube
sourcelon = lyr_cube.coord('source_longitude').points[0]
sourcelat = lyr_cube.coord('source_latitude').points[0]
if self.rsmc:
rsmc_cube = self.layer_list[0].cube
plt.plot(sourcelon, sourcelat, 'ks',
markersize=6, label='Source Location',
transform=ccrs.PlateCarree())
maxval = np.argmax(rsmc_cube.data)
xyvals = np.unravel_index(maxval, rsmc_cube.data.shape)
lon_max = rsmc_cube.coord('longitude').points[xyvals[0]]
lat_max = rsmc_cube.coord('latitude').points[xyvals[1]]
if rsmc_cube.attributes['short_name'] != 'TimeOfArrival':
plt.plot(lon_max, lat_max, 'k*',
transform=ccrs.PlateCarree())
else:
plt.plot(sourcelon, sourcelat, 'r*',
markersize=10, label='Source Location',
transform=ccrs.PlateCarree())
# Add a title
if self.title is None:
self.title = self.gen_title()
elif self.title == 'name_verbose':
self.title = self.gen_name_verbose_title()
# Add prefix to title if required
if self.titleprefix is not None:
self.title = self.titleprefix + self.title
plt.title(self.title, linespacing=1.5)
# Add a supertitle
if self.suptitle is not None:
plt.suptitle(self.suptitle, fontsize=16)
# Add branding
if self.mobrand:
insert_logo()
copyright_sign = unicodedata.lookup('COPYRIGHT SIGN')
copyright_txt = copyright_sign + " Met Office Crown Copyright"
plt.annotate(copyright_txt,
(0, 0), (-50, -50),
xycoords='axes fraction',
textcoords='offset points',
va='top')
# Ensure that the field_plot object has all the changes made here
# included in self.fig:
self.fig = fig
return self.fig
