def __repr__(self): r''' # note that we use raw strings to avoid having to use double back slashes below NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function:: utf8.__repr__() works same as str.repr() when processing ascii string >>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'" True >>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\'' True >>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"' True >>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\'' True >>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n True Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string:: >>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字') True >>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字') True >>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字") True >>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字") True >>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n True ''' if str.find(self, "'") >= 0 and str.find(self, '"') < 0: # only single quote exists return '"' + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"' else: return "'" + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'"
def sort_key(s): """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/) is used for utf-8 and unicode strings sorting and for utf-8 strings comparison Note: pyuca is a very memory cost module! It loads the whole "allkey.txt" file (~2mb!) into the memory. But this functionality is needed only when sort_key() is called as a part of sort() function or when Utf8 strings are compared. So, it is a lazy "sort_key" function which (ONLY ONCE, ON ITS FIRST CALL) imports pyuca and replaces itself with a real sort_key() function """ global sort_key try: from gluon.contrib.pyuca import unicode_collator unicode_sort_key = unicode_collator.sort_key sort_key = lambda s: unicode_sort_key( to_unicode(s, 'utf-8') if isinstance(s, str) else s) except: sort_key = lambda s: ( to_unicode(s, 'utf-8') if isinstance(s, str) else s).lower() return sort_key(s)
def write_file(file, contents): file.write('# -*- coding: utf-8 -*-\n{\n') for key in sorted(contents, key = sort_function): file.write('%s: %s,\n' % (repr(to_unicode(key)), repr(to_unicode(contents[key])))) file.write('}\n') file.close()
def truncate(string, length, dots='...'): """Returns string of length < *length* or truncate string with adding *dots* suffix to the string's end Args: length (int): max length of string dots (str or unicode): string suffix, when string is cutted Returns: (utf8-str): original or cutted string """ text = to_unicode(string, 'utf-8') dots = to_unicode(dots, 'utf-8') if isinstance(dots, str) else dots if len(text) > length: text = text[:length - len(dots)] + dots return str.__new__(Utf8, text.encode('utf-8'))
def chapter(): try: from gluon._compat import to_bytes, to_native, to_unicode except ImportError: redirect(URL('index', vars=dict(FLASH_MSG = "CompatError"))) book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0) subfolder = get_subfolder(book_id) info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE) chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE) chapter_title = chapters[chapter_id][1] response.title = '%s - %s' % (info['title'], to_unicode(chapter_title)) filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id) dest = os.path.join(request.folder, 'static_chaps', subfolder, '%.2i.html' % chapter_id) if not FORCE_RENDER: response.headers['Cache-Control'] = 'public, must-revalidate' response.headers['Expires'] = calc_date() response.headers['Pragma'] = None if (not os.path.isfile(dest)) or FORCE_RENDER: try: content = open(filename, 'rt', encoding='utf-8').read() except FileNotFoundError: redirect(URL('index', vars=dict(FLASH_MSG = 'FileNotFoundError'))) content = convert2html(book_id, content).xml() if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) open(dest, 'wb').write(content) content = XML(content) return locals() else: content = XML(open(dest, 'rt', encoding='utf-8').read()) return locals()
def test_plain(self): T = languages.translator(self.langpath, self.http_accept_language) self.assertEqual(str(T('Hello World')), 'Hello World') self.assertEqual(str(T('Hello World## comment')), 'Hello World') self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops') self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops') self.assertEqual(str(T('%s %%{quark[0]}', 1)), '1 quark') self.assertEqual(str(T('%s %%{quark[0]}', 2)), '2 quarks') self.assertEqual(str(T.M('**Hello World**')), '<strong>Hello World</strong>') T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo') self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
def __new__(cls, content='', codepage='utf-8'): if isinstance(content, unicodeT): return str.__new__(cls, to_native(content, 'utf-8')) elif codepage in ('utf-8', 'utf8') or isinstance(content, cls): return str.__new__(cls, content) else: return str.__new__(cls, to_native(to_unicode(content, codepage), 'utf-8'))
def ord(char): """Returns unicode id for utf8 or unicode *char* character SUPPOSE that *char* is an utf-8 or unicode character only """ if isinstance(char, unicodeT): return __builtin__.ord(char) return __builtin__.ord(to_unicode(char, 'utf-8'))
def __new__(cls, content="", codepage="utf-8"): if isinstance(content, unicodeT): return str.__new__(cls, to_native(content, "utf-8")) elif codepage in ("utf-8", "utf8") or isinstance(content, cls): return str.__new__(cls, content) else: return str.__new__(cls, to_native(to_unicode(content, codepage), "utf-8"))
def __new__(cls, content='', codepage='utf-8'): if isinstance(content, unicodeT): return str.__new__(cls, to_native(content, 'utf-8')) elif codepage in ('utf-8', 'utf8') or isinstance(content, cls): return str.__new__(cls, content) else: return str.__new__( cls, to_native(to_unicode(content, codepage), 'utf-8'))
def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attributes): if not text: test = '' if 'extras' in attributes: extras = attributes['extras'] del attributes['extras'] else: extras=None text = to_unicode(text, encoding, 'replace') return XML(markdown(text,extras=extras, safe_mode=safe_mode, html4tags=html4tags)\ .encode(encoding,'xmlcharrefreplace'),**attributes)
def __repr__(self): r''' # note that we use raw strings to avoid having to use double back slashes below NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function:: utf8.__repr__() works same as str.repr() when processing ascii string >>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'" True >>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\'' True >>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"' True >>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\'' True >>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n True Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string:: >>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字') True >>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字') True >>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字") True >>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字") True >>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n True ''' if str.find(self, "'") >= 0 and str.find( self, '"') < 0: # only single quote exists return '"' + to_native( to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"' else: return "'" + to_native( to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'"
def search(): def fix_relative_link(match): return "%s%s%s%s" % ( match.group(1), '../chapter/', book_id, match.group(3) ) # link rewritten to be relative to the search URL try: from gluon._compat import to_bytes, to_native, to_unicode except ImportError: redirect(URL('index', vars=dict(FLASH_MSG="CompatError"))) book_id = request.args(0) or redirect(URL('index')) search = request.vars.search or redirect(URL('chapter', args=book_id)) subfolder = get_subfolder(book_id) info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE) chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE) results = [] content = H2('No results for "%s"' % search) relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])') for chapter in chapters: chapter_id = int(chapter[0]) filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id) data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '') k = data.lower().find(to_unicode(search).lower()) if k >= 0: snippet = data[data.rfind('\n\n', 0, k) + 1:data.find('\n\n', k)].strip() snippet = relative_link_re.sub(fix_relative_link, snippet) results.append((chapter[0], chapter[1], chapter[2], convert2html(book_id, snippet))) content = CAT(*[ DIV( H2( A(chapter[1], _href=URL('chapter', vars=dict(search=search), args=(book_id, chapter[0], chapter[2])))), chapter[3], BR(), A('more', _href=URL('chapter', vars=dict(search=search), args=(book_id, chapter[0], chapter[2])), _class="btn")) for chapter in results ]) response.view = 'default/chapter.html' return locals()
def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attributes): if not text: test = '' if 'extras' in attributes: extras = attributes['extras'] del attributes['extras'] else: extras = None text = to_unicode(text, encoding, 'replace') return XML(markdown(text,extras=extras, safe_mode=safe_mode, html4tags=html4tags)\ .encode(encoding,'xmlcharrefreplace'),**attributes)
def write_dict(filename, contents): if '__corrupted__' in contents: return fp = None try: fp = LockedFile(filename, 'w') fp.write('# -*- coding: utf-8 -*-\n{\n') for key in sorted(contents, key=lambda x: to_unicode(x, 'utf-8').lower()): fp.write('%s: %s,\n' % (repr(Utf8(key)), repr(Utf8(contents[key])))) fp.write('}\n') except (IOError, OSError): if is_writable(): logging.warning('Unable to write to file %s' % filename) return finally: if fp: fp.close()
def search(): def fix_relative_link(match): return "%s%s%s%s" % (match.group(1), '../chapter/', book_id, match.group(3)) # link rewritten to be relative to the search URL try: from gluon._compat import to_bytes, to_native, to_unicode except ImportError: redirect(URL('index', vars=dict(FLASH_MSG = "CompatError"))) book_id = request.args(0) or redirect(URL('index')) search = request.vars.search or redirect(URL('chapter', args=book_id)) subfolder = get_subfolder(book_id) info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE) chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE) results = [] content = H2('No results for "%s"' % search) relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])') for chapter in chapters: chapter_id = int(chapter[0]) filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id) data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '') k = data.lower().find(to_unicode(search).lower()) if k >= 0: snippet = data[data.rfind('\n\n', 0, k) + 1:data.find('\n\n', k)].strip() snippet = relative_link_re.sub(fix_relative_link, snippet) results.append((chapter[0], chapter[1], chapter[2], convert2html(book_id, snippet))) content = CAT(*[DIV(H2(A(chapter[1], _href=URL('chapter', vars=dict(search=search), args=(book_id, chapter[0], chapter[2])))), chapter[3], BR(), A('more', _href=URL('chapter', vars=dict(search=search), args=(book_id, chapter[0], chapter[2])), _class="btn")) for chapter in results]) response.view = 'default/chapter.html' return locals()
def chapter(): try: from gluon._compat import to_bytes, to_native, to_unicode except ImportError: redirect(URL('index', vars=dict(FLASH_MSG="CompatError"))) book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0) subfolder = get_subfolder(book_id) info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE) chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE) chapter_title = chapters[chapter_id][1] response.title = '%s - %s' % (info['title'], to_unicode(chapter_title)) filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id) dest = os.path.join(request.folder, 'static_chaps', subfolder, '%.2i.html' % chapter_id) if not FORCE_RENDER: response.headers['Cache-Control'] = 'public, must-revalidate' response.headers['Expires'] = calc_date() response.headers['Pragma'] = None if (not os.path.isfile(dest)) or FORCE_RENDER: try: content = open(filename, 'rt', encoding='utf-8').read() except FileNotFoundError: redirect(URL('index', vars=dict(FLASH_MSG='FileNotFoundError'))) content = convert2html(book_id, content).xml() if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) open(dest, 'wb').write(content) content = XML(content) return locals() else: content = XML(open(dest, 'rt', encoding='utf-8').read()) return locals()
def sub_dict(m): """ word(key or num) !word(key or num), !!word(key or num), !!!word(key or num) ?word1?word(key or num) ??word(key or num), ?word(key or num) ?word1?word?word0(key or num) ?word1?word?(key or num) ??word?word0(key or num) ?word1?word?(key or num) ??word?(key or num), ?word?(key or num) """ w, n = m.group('w', 'n') c = w[0] n = int(n) if n.isdigit() else symbols[n] if c not in '!?': return self.plural(w, n) elif c == '?': # ?[word1]?word[?word0](key or num), ?[word1]?word(key or num) or ?word(key or num) (p1, sep, p2) = w[1:].partition("?") part1 = p1 if sep else "" (part2, sep, part3) = (p2 if sep else p1).partition("?") if not sep: part3 = part2 num = int(n) return part1 if num == 1 else part3 if num == 0 else part2 elif w.startswith('!!!'): word = w[3:] fun = upper_fun elif w.startswith('!!'): word = w[2:] fun = title_fun else: word = w[1:] fun = cap_fun s = fun(self.plural(word, n)) return s if PY2 else to_unicode(s)
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) | Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com> | for Web2py project Utilities and class for UTF8 strings managing ---------------------------------------------- """ from __future__ import print_function from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload __all__ = ['Utf8'] repr_escape_tab = {} #FIXME PY3 for i in range(1, 32): repr_escape_tab[i] = to_unicode("\\"+"x%02x" % i) repr_escape_tab[7] = u'\\a' repr_escape_tab[8] = u'\\b' repr_escape_tab[9] = u'\\t' repr_escape_tab[10] = u'\\n' repr_escape_tab[11] = u'\\v' repr_escape_tab[12] = u'\\f' repr_escape_tab[13] = u'\\r' repr_escape_tab[ord('\\')] = u'\\\\' repr_escape_tab2 = repr_escape_tab.copy() repr_escape_tab2[ord('\'')] = u"\\'" def sort_key(s): """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/) is used for utf-8 and unicode strings sorting and for utf-8 strings
def test_decode(self): T = languages.translator(self.langpath, self.http_accept_language) messages = Messages(T) messages.update({'email_sent': 'Email sent', 'test': "ä"}) self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
def test_plain(self): T = languages.translator(self.langpath, self.http_accept_language) self.assertEqual(str(T('Hello World')), 'Hello World') self.assertEqual(str(T('Hello World## comment')), 'Hello World') self.assertEqual(str(T.M('**Hello World**')), '<strong>Hello World</strong>') # sub_tuple testing self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops') self.assertEqual(str(T('%%{quark(%s)}', 1)), 'quark') self.assertEqual(str(T('%%{quark(%i)}', 2)), 'quarks') self.assertEqual(str(T('%%{!quark(%s)}', 1)), 'Quark') self.assertEqual(str(T('%%{!!quark(%i)}', 2)), 'Quarks') self.assertEqual(str(T('%%{!!!quark(%s)}', 0)), 'QUARKS') self.assertEqual(str(T('%%{?an?%i}', 1)), 'an') self.assertEqual(str(T('%%{?an?%s}', 0)), '0') self.assertEqual(str(T('%%{??%i}', 1)), '') self.assertEqual(str(T('%%{??%s}', 2)), '2') self.assertEqual(str(T('%%{?%i}', 1)), '') self.assertEqual(str(T('%%{?%s}', 0)), '0') self.assertEqual(str(T('%%{?one?%i?zero}', 1)), 'one') self.assertEqual(str(T('%%{?one?%s?zero}', 23)), '23') self.assertEqual(str(T('%%{?one?%i?zero}', 0)), 'zero') self.assertEqual(str(T('%%{?one?%s?}', 1)), 'one') self.assertEqual(str(T('%%{?one?%i?}', 23)), '23') self.assertEqual(str(T('%%{?one?%s?}', 0)), '') self.assertEqual(str(T('%%{??%i?zero}', 1)), '') self.assertEqual(str(T('%%{??%s?zero}', 23)), '23') self.assertEqual(str(T('%%{??%i?zero}', 0)), 'zero') self.assertEqual(str(T('%%{??1?}%s', '')), '') self.assertEqual(str(T('%%{??%s?}', 23)), '23') self.assertEqual(str(T('%%{??0?}%s', '')), '') self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops') self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)), '1 one') self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)), '2 not_one') self.assertEqual(str(T('%%{??on[0]} %i', 1)), ' 1') self.assertEqual(str(T('%%{??on[0]} %s', 0)), 'on 0') self.assertEqual(str(T('%%{?on[0]} %s', 1)), ' 1') self.assertEqual(str(T('%%{?on[0]} %i', 2)), 'on 2') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)), '1 one') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)), '2 or_more') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)), '0 zero') self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)), '1 one') self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)), '2 hands') self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)), '0 ') self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)), '1 ') self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)), '2 or_more') self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)), '0 zero') self.assertEqual(str(T('%i%%{??nd?[0]}', 1)), '1') self.assertEqual(str(T('%i%%{??nd?[0]}', 2)), '2nd') self.assertEqual(str(T('%i%%{??nd?[0]}', 0)), '0') self.assertEqual(str(T('%i%%{?st?[0]}', 1)), '1st') self.assertEqual(str(T('%i%%{?st?[0]}', 2)), '2') self.assertEqual(str(T('%i%%{?st?[0]}', 0)), '0') # sub_dict testing self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))), '1 is') self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))), '2 are') self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))), 'ARE') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))), '2 ') self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))), '0 ') self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual( str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))), '1 one') self.assertEqual( str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))), '4 other') self.assertEqual( str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))), '0 zero') self.assertEqual( str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))), '1 one') self.assertEqual( str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))), '2 two_or_more') self.assertEqual( str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))), '0 ') self.assertEqual( str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))), '1 ') self.assertEqual( str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))), '2 two_or_more') self.assertEqual( str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))), '0 zero') self.assertEqual( str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))), '1 ') self.assertEqual( str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))), '0 ') self.assertEqual( str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))), '2 two_or_more') T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo') self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
def test_decode(self): T = languages.translator(self.langpath, self.http_accept_language) messages = Messages(T) messages.update({'email_sent':'Email sent', 'test': "ä"}) self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
def __getitem__(self, index): return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
def __getslice__(self, begin, end): return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
def decoder(buffer): encoding = autoDetectXMLEncoding(buffer) return to_unicode(buffer, charset=encoding)
def upper_fun(s): return to_bytes(to_unicode(s).upper())
def cap_fun(s): return to_bytes(to_unicode(s).capitalize())
def encode(self, *a, **b): if PY2 and a[0] != 'utf8': return to_unicode(str(self)).encode(*a, **b) else: return str(self)
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) | Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com> | for Web2py project Utilities and class for UTF8 strings managing ---------------------------------------------- """ from __future__ import print_function from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload __all__ = ['Utf8'] repr_escape_tab = {} #FIXME PY3 for i in range(1, 32): repr_escape_tab[i] = to_unicode("\\" + "x%02x" % i) repr_escape_tab[7] = u'\\a' repr_escape_tab[8] = u'\\b' repr_escape_tab[9] = u'\\t' repr_escape_tab[10] = u'\\n' repr_escape_tab[11] = u'\\v' repr_escape_tab[12] = u'\\f' repr_escape_tab[13] = u'\\r' repr_escape_tab[ord('\\')] = u'\\\\' repr_escape_tab2 = repr_escape_tab.copy() repr_escape_tab2[ord('\'')] = u"\\'" def sort_key(s): """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/) is used for utf-8 and unicode strings sorting and for utf-8 strings
def sort_function(x): return to_unicode(x, 'utf-8').lower()
def __len__(self): return len(to_unicode(self, "utf-8"))
def title_fun(s): return to_bytes(to_unicode(s).title())
def __getslice__(self, begin, end): return str.__new__( Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
def __len__(self): return len(to_unicode(self, 'utf-8'))
def test_plain(self): T = languages.translator(self.langpath, self.http_accept_language) self.assertEqual(str(T('Hello World')), 'Hello World') self.assertEqual(str(T('Hello World## comment')), 'Hello World') self.assertEqual(str(T.M('**Hello World**')), '<strong>Hello World</strong>') # sub_tuple testing self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops') self.assertEqual(str(T('%%{quark(%s)}', 1)), 'quark') self.assertEqual(str(T('%%{quark(%i)}', 2)), 'quarks') self.assertEqual(str(T('%%{!quark(%s)}', 1)), 'Quark') self.assertEqual(str(T('%%{!!quark(%i)}', 2)), 'Quarks') self.assertEqual(str(T('%%{!!!quark(%s)}', 0)), 'QUARKS') self.assertEqual(str(T('%%{?an?%i}', 1)), 'an') self.assertEqual(str(T('%%{?an?%s}', 0)), '0') self.assertEqual(str(T('%%{??%i}', 1)), '') self.assertEqual(str(T('%%{??%s}', 2)), '2') self.assertEqual(str(T('%%{?%i}', 1)), '') self.assertEqual(str(T('%%{?%s}', 0)), '0') self.assertEqual(str(T('%%{?one?%i?zero}', 1)), 'one') self.assertEqual(str(T('%%{?one?%s?zero}', 23)), '23') self.assertEqual(str(T('%%{?one?%i?zero}', 0)), 'zero') self.assertEqual(str(T('%%{?one?%s?}', 1)), 'one') self.assertEqual(str(T('%%{?one?%i?}', 23)), '23') self.assertEqual(str(T('%%{?one?%s?}', 0)), '') self.assertEqual(str(T('%%{??%i?zero}', 1)), '') self.assertEqual(str(T('%%{??%s?zero}', 23)), '23') self.assertEqual(str(T('%%{??%i?zero}', 0)), 'zero') self.assertEqual(str(T('%%{??1?}%s', '')), '') self.assertEqual(str(T('%%{??%s?}', 23)), '23') self.assertEqual(str(T('%%{??0?}%s', '')), '') self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop') self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops') self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)), '1 one') self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)), '2 not_one') self.assertEqual(str(T('%%{??on[0]} %i', 1)), ' 1') self.assertEqual(str(T('%%{??on[0]} %s', 0)), 'on 0') self.assertEqual(str(T('%%{?on[0]} %s', 1)), ' 1') self.assertEqual(str(T('%%{?on[0]} %i', 2)), 'on 2') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)), '1 one') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)), '2 or_more') self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)), '0 zero') self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)), '1 one') self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)), '2 hands') self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)), '0 ') self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)), '1 ') self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)), '2 or_more') self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)), '0 zero') self.assertEqual(str(T('%i%%{??nd?[0]}', 1)), '1') self.assertEqual(str(T('%i%%{??nd?[0]}', 2)), '2nd') self.assertEqual(str(T('%i%%{??nd?[0]}', 0)), '0') self.assertEqual(str(T('%i%%{?st?[0]}', 1)), '1st') self.assertEqual(str(T('%i%%{?st?[0]}', 2)), '2') self.assertEqual(str(T('%i%%{?st?[0]}', 0)), '0') # sub_dict testing self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))), '1 is') self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))), '2 are') self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))), 'ARE') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))), '2 ') self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))), '0 ') self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))), '2 not_one') self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))), '0 not_one') self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))), '4 other') self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))), '0 zero') self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))), '1 one') self.assertEqual(str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))), '2 two_or_more') self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))), '0 ') self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))), '2 two_or_more') self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))), '0 zero') self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))), '1 ') self.assertEqual(str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))), '0 ') self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))), '2 two_or_more') T.force('it') self.assertEqual(str(T('Hello World')), 'Salve Mondo') self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
def __getitem__(self, index): return str.__new__( Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))