def __repr__(self):
r''' # note that we use raw strings to avoid having to use double back slashes below
NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function::
utf8.__repr__() works same as str.repr() when processing ascii string
>>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'"
True
>>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\''
True
>>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"'
True
>>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\''
True
>>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
True
Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string::
>>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字')
True
>>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字')
True
>>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字")
True
>>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
True
>>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
True
'''
if str.find(self, "'") >= 0 and str.find(self, '"') < 0: # only single quote exists
return '"' + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"'
else:
return "'" + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'"
def sort_key(s):
"""Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
is used for utf-8 and unicode strings sorting and for utf-8 strings
comparison
Note:
pyuca is a very memory cost module! It loads the whole
"allkey.txt" file (~2mb!) into the memory. But this
functionality is needed only when sort_key() is called as a
part of sort() function or when Utf8 strings are compared.
So, it is a lazy "sort_key" function which (ONLY ONCE, ON ITS
FIRST CALL) imports pyuca and replaces itself with a real
sort_key() function
"""
global sort_key
try:
from gluon.contrib.pyuca import unicode_collator
unicode_sort_key = unicode_collator.sort_key
sort_key = lambda s: unicode_sort_key(
to_unicode(s, 'utf-8') if isinstance(s, str) else s)
except:
sort_key = lambda s: (
to_unicode(s, 'utf-8') if isinstance(s, str) else s).lower()
return sort_key(s)
def write_file(file, contents):
file.write('# -*- coding: utf-8 -*-\n{\n')
for key in sorted(contents, key = sort_function):
file.write('%s: %s,\n' % (repr(to_unicode(key)),
repr(to_unicode(contents[key]))))
file.write('}\n')
file.close()
def truncate(string, length, dots='...'):
"""Returns string of length < *length* or truncate string with adding
*dots* suffix to the string's end
Args:
length (int): max length of string
dots (str or unicode): string suffix, when string is cutted
Returns:
(utf8-str): original or cutted string
"""
text = to_unicode(string, 'utf-8')
dots = to_unicode(dots, 'utf-8') if isinstance(dots, str) else dots
if len(text) > length:
text = text[:length - len(dots)] + dots
return str.__new__(Utf8, text.encode('utf-8'))
def chapter():
try:
from gluon._compat import to_bytes, to_native, to_unicode
except ImportError:
redirect(URL('index', vars=dict(FLASH_MSG = "CompatError")))
book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0)
subfolder = get_subfolder(book_id)
info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE)
chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE)
chapter_title = chapters[chapter_id][1]
response.title = '%s - %s' % (info['title'], to_unicode(chapter_title))
filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
dest = os.path.join(request.folder, 'static_chaps', subfolder, '%.2i.html' % chapter_id)
if not FORCE_RENDER:
response.headers['Cache-Control'] = 'public, must-revalidate'
response.headers['Expires'] = calc_date()
response.headers['Pragma'] = None
if (not os.path.isfile(dest)) or FORCE_RENDER:
try:
content = open(filename, 'rt', encoding='utf-8').read()
except FileNotFoundError:
redirect(URL('index', vars=dict(FLASH_MSG = 'FileNotFoundError')))
content = convert2html(book_id, content).xml()
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
open(dest, 'wb').write(content)
content = XML(content)
return locals()
else:
content = XML(open(dest, 'rt', encoding='utf-8').read())
return locals()
def test_plain(self):
T = languages.translator(self.langpath, self.http_accept_language)
self.assertEqual(str(T('Hello World')),
'Hello World')
self.assertEqual(str(T('Hello World## comment')),
'Hello World')
self.assertEqual(str(T('%s %%{shop}', 1)),
'1 shop')
self.assertEqual(str(T('%s %%{shop}', 2)),
'2 shops')
self.assertEqual(str(T('%s %%{shop[0]}', 1)),
'1 shop')
self.assertEqual(str(T('%s %%{shop[0]}', 2)),
'2 shops')
self.assertEqual(str(T('%s %%{quark[0]}', 1)),
'1 quark')
self.assertEqual(str(T('%s %%{quark[0]}', 2)),
'2 quarks')
self.assertEqual(str(T.M('**Hello World**')),
'<strong>Hello World</strong>')
T.force('it')
self.assertEqual(str(T('Hello World')),
'Salve Mondo')
self.assertEqual(to_unicode(T('Hello World')),
'Salve Mondo')
def __new__(cls, content='', codepage='utf-8'):
if isinstance(content, unicodeT):
return str.__new__(cls, to_native(content, 'utf-8'))
elif codepage in ('utf-8', 'utf8') or isinstance(content, cls):
return str.__new__(cls, content)
else:
return str.__new__(cls, to_native(to_unicode(content, codepage), 'utf-8'))
def ord(char):
"""Returns unicode id for utf8 or unicode *char* character
SUPPOSE that *char* is an utf-8 or unicode character only
"""
if isinstance(char, unicodeT):
return __builtin__.ord(char)
return __builtin__.ord(to_unicode(char, 'utf-8'))
def __new__(cls, content="", codepage="utf-8"):
if isinstance(content, unicodeT):
return str.__new__(cls, to_native(content, "utf-8"))
elif codepage in ("utf-8", "utf8") or isinstance(content, cls):
return str.__new__(cls, content)
else:
return str.__new__(cls, to_native(to_unicode(content, codepage), "utf-8"))
def __new__(cls, content='', codepage='utf-8'):
if isinstance(content, unicodeT):
return str.__new__(cls, to_native(content, 'utf-8'))
elif codepage in ('utf-8', 'utf8') or isinstance(content, cls):
return str.__new__(cls, content)
else:
return str.__new__(
cls, to_native(to_unicode(content, codepage), 'utf-8'))
def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attributes):
if not text:
test = ''
if 'extras' in attributes:
extras = attributes['extras']
del attributes['extras']
else:
extras=None
text = to_unicode(text, encoding, 'replace')
return XML(markdown(text,extras=extras,
safe_mode=safe_mode, html4tags=html4tags)\
.encode(encoding,'xmlcharrefreplace'),**attributes)
def __repr__(self):
r''' # note that we use raw strings to avoid having to use double back slashes below
NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function::
utf8.__repr__() works same as str.repr() when processing ascii string
>>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'"
True
>>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\''
True
>>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"'
True
>>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\''
True
>>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
True
Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string::
>>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字')
True
>>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字')
True
>>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字")
True
>>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
True
>>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
True
'''
if str.find(self, "'") >= 0 and str.find(
self, '"') < 0: # only single quote exists
return '"' + to_native(
to_unicode(self, 'utf-8').translate(repr_escape_tab),
'utf-8') + '"'
else:
return "'" + to_native(
to_unicode(self, 'utf-8').translate(repr_escape_tab2),
'utf-8') + "'"
def search():
def fix_relative_link(match):
return "%s%s%s%s" % (
match.group(1), '../chapter/', book_id, match.group(3)
) # link rewritten to be relative to the search URL
try:
from gluon._compat import to_bytes, to_native, to_unicode
except ImportError:
redirect(URL('index', vars=dict(FLASH_MSG="CompatError")))
book_id = request.args(0) or redirect(URL('index'))
search = request.vars.search or redirect(URL('chapter', args=book_id))
subfolder = get_subfolder(book_id)
info = cache.ram('info_%s' % subfolder,
lambda: get_info(subfolder),
time_expire=TIME_EXPIRE)
chapters = cache.ram('chapters_%s' % subfolder,
lambda: get_chapters(subfolder),
time_expire=TIME_EXPIRE)
results = []
content = H2('No results for "%s"' % search)
relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])')
for chapter in chapters:
chapter_id = int(chapter[0])
filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '')
k = data.lower().find(to_unicode(search).lower())
if k >= 0:
snippet = data[data.rfind('\n\n', 0, k) +
1:data.find('\n\n', k)].strip()
snippet = relative_link_re.sub(fix_relative_link, snippet)
results.append((chapter[0], chapter[1], chapter[2],
convert2html(book_id, snippet)))
content = CAT(*[
DIV(
H2(
A(chapter[1],
_href=URL('chapter',
vars=dict(search=search),
args=(book_id, chapter[0],
chapter[2])))), chapter[3], BR(),
A('more',
_href=URL('chapter',
vars=dict(search=search),
args=(book_id, chapter[0], chapter[2])),
_class="btn")) for chapter in results
])
response.view = 'default/chapter.html'
return locals()
def test_plain(self):
T = languages.translator(self.langpath, self.http_accept_language)
self.assertEqual(str(T('Hello World')), 'Hello World')
self.assertEqual(str(T('Hello World## comment')), 'Hello World')
self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop')
self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops')
self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop')
self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops')
self.assertEqual(str(T('%s %%{quark[0]}', 1)), '1 quark')
self.assertEqual(str(T('%s %%{quark[0]}', 2)), '2 quarks')
self.assertEqual(str(T.M('**Hello World**')),
'<strong>Hello World</strong>')
T.force('it')
self.assertEqual(str(T('Hello World')), 'Salve Mondo')
self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
def WIKI(text,
encoding="utf8",
safe_mode='escape',
html4tags=False,
**attributes):
if not text:
test = ''
if 'extras' in attributes:
extras = attributes['extras']
del attributes['extras']
else:
extras = None
text = to_unicode(text, encoding, 'replace')
return XML(markdown(text,extras=extras,
safe_mode=safe_mode, html4tags=html4tags)\
.encode(encoding,'xmlcharrefreplace'),**attributes)
def write_dict(filename, contents):
if '__corrupted__' in contents:
return
fp = None
try:
fp = LockedFile(filename, 'w')
fp.write('# -*- coding: utf-8 -*-\n{\n')
for key in sorted(contents, key=lambda x: to_unicode(x, 'utf-8').lower()):
fp.write('%s: %s,\n' % (repr(Utf8(key)),
repr(Utf8(contents[key]))))
fp.write('}\n')
except (IOError, OSError):
if is_writable():
logging.warning('Unable to write to file %s' % filename)
return
finally:
if fp:
fp.close()
def search():
def fix_relative_link(match):
return "%s%s%s%s" % (match.group(1),
'../chapter/',
book_id,
match.group(3)) # link rewritten to be relative to the search URL
try:
from gluon._compat import to_bytes, to_native, to_unicode
except ImportError:
redirect(URL('index', vars=dict(FLASH_MSG = "CompatError")))
book_id = request.args(0) or redirect(URL('index'))
search = request.vars.search or redirect(URL('chapter', args=book_id))
subfolder = get_subfolder(book_id)
info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE)
chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE)
results = []
content = H2('No results for "%s"' % search)
relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])')
for chapter in chapters:
chapter_id = int(chapter[0])
filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '')
k = data.lower().find(to_unicode(search).lower())
if k >= 0:
snippet = data[data.rfind('\n\n', 0, k) + 1:data.find('\n\n', k)].strip()
snippet = relative_link_re.sub(fix_relative_link, snippet)
results.append((chapter[0], chapter[1], chapter[2], convert2html(book_id, snippet)))
content = CAT(*[DIV(H2(A(chapter[1],
_href=URL('chapter',
vars=dict(search=search),
args=(book_id, chapter[0], chapter[2])))),
chapter[3], BR(),
A('more', _href=URL('chapter',
vars=dict(search=search),
args=(book_id, chapter[0], chapter[2])), _class="btn"))
for chapter in results])
response.view = 'default/chapter.html'
return locals()
def chapter():
try:
from gluon._compat import to_bytes, to_native, to_unicode
except ImportError:
redirect(URL('index', vars=dict(FLASH_MSG="CompatError")))
book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0)
subfolder = get_subfolder(book_id)
info = cache.ram('info_%s' % subfolder,
lambda: get_info(subfolder),
time_expire=TIME_EXPIRE)
chapters = cache.ram('chapters_%s' % subfolder,
lambda: get_chapters(subfolder),
time_expire=TIME_EXPIRE)
chapter_title = chapters[chapter_id][1]
response.title = '%s - %s' % (info['title'], to_unicode(chapter_title))
filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
dest = os.path.join(request.folder, 'static_chaps', subfolder,
'%.2i.html' % chapter_id)
if not FORCE_RENDER:
response.headers['Cache-Control'] = 'public, must-revalidate'
response.headers['Expires'] = calc_date()
response.headers['Pragma'] = None
if (not os.path.isfile(dest)) or FORCE_RENDER:
try:
content = open(filename, 'rt', encoding='utf-8').read()
except FileNotFoundError:
redirect(URL('index', vars=dict(FLASH_MSG='FileNotFoundError')))
content = convert2html(book_id, content).xml()
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
open(dest, 'wb').write(content)
content = XML(content)
return locals()
else:
content = XML(open(dest, 'rt', encoding='utf-8').read())
return locals()
def sub_dict(m):
""" word(key or num)
!word(key or num), !!word(key or num), !!!word(key or num)
?word1?word(key or num)
??word(key or num), ?word(key or num)
?word1?word?word0(key or num)
?word1?word?(key or num)
??word?word0(key or num)
?word1?word?(key or num)
??word?(key or num), ?word?(key or num)
"""
w, n = m.group('w', 'n')
c = w[0]
n = int(n) if n.isdigit() else symbols[n]
if c not in '!?':
return self.plural(w, n)
elif c == '?':
# ?[word1]?word[?word0](key or num), ?[word1]?word(key or num) or ?word(key or num)
(p1, sep, p2) = w[1:].partition("?")
part1 = p1 if sep else ""
(part2, sep, part3) = (p2 if sep else p1).partition("?")
if not sep:
part3 = part2
num = int(n)
return part1 if num == 1 else part3 if num == 0 else part2
elif w.startswith('!!!'):
word = w[3:]
fun = upper_fun
elif w.startswith('!!'):
word = w[2:]
fun = title_fun
else:
word = w[1:]
fun = cap_fun
s = fun(self.plural(word, n))
return s if PY2 else to_unicode(s)
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
| Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com>
| for Web2py project
Utilities and class for UTF8 strings managing
----------------------------------------------
"""
from __future__ import print_function
from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload
__all__ = ['Utf8']
repr_escape_tab = {}
#FIXME PY3
for i in range(1, 32):
repr_escape_tab[i] = to_unicode("\\"+"x%02x" % i)
repr_escape_tab[7] = u'\\a'
repr_escape_tab[8] = u'\\b'
repr_escape_tab[9] = u'\\t'
repr_escape_tab[10] = u'\\n'
repr_escape_tab[11] = u'\\v'
repr_escape_tab[12] = u'\\f'
repr_escape_tab[13] = u'\\r'
repr_escape_tab[ord('\\')] = u'\\\\'
repr_escape_tab2 = repr_escape_tab.copy()
repr_escape_tab2[ord('\'')] = u"\\'"
def sort_key(s):
"""Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
is used for utf-8 and unicode strings sorting and for utf-8 strings
def test_decode(self):
T = languages.translator(self.langpath, self.http_accept_language)
messages = Messages(T)
messages.update({'email_sent': 'Email sent', 'test': "ä"})
self.assertEqual(to_unicode(messages.email_sent, 'utf-8'),
'Email sent')
def test_plain(self):
T = languages.translator(self.langpath, self.http_accept_language)
self.assertEqual(str(T('Hello World')), 'Hello World')
self.assertEqual(str(T('Hello World## comment')), 'Hello World')
self.assertEqual(str(T.M('**Hello World**')),
'<strong>Hello World</strong>')
# sub_tuple testing
self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop')
self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops')
self.assertEqual(str(T('%%{quark(%s)}', 1)), 'quark')
self.assertEqual(str(T('%%{quark(%i)}', 2)), 'quarks')
self.assertEqual(str(T('%%{!quark(%s)}', 1)), 'Quark')
self.assertEqual(str(T('%%{!!quark(%i)}', 2)), 'Quarks')
self.assertEqual(str(T('%%{!!!quark(%s)}', 0)), 'QUARKS')
self.assertEqual(str(T('%%{?an?%i}', 1)), 'an')
self.assertEqual(str(T('%%{?an?%s}', 0)), '0')
self.assertEqual(str(T('%%{??%i}', 1)), '')
self.assertEqual(str(T('%%{??%s}', 2)), '2')
self.assertEqual(str(T('%%{?%i}', 1)), '')
self.assertEqual(str(T('%%{?%s}', 0)), '0')
self.assertEqual(str(T('%%{?one?%i?zero}', 1)), 'one')
self.assertEqual(str(T('%%{?one?%s?zero}', 23)), '23')
self.assertEqual(str(T('%%{?one?%i?zero}', 0)), 'zero')
self.assertEqual(str(T('%%{?one?%s?}', 1)), 'one')
self.assertEqual(str(T('%%{?one?%i?}', 23)), '23')
self.assertEqual(str(T('%%{?one?%s?}', 0)), '')
self.assertEqual(str(T('%%{??%i?zero}', 1)), '')
self.assertEqual(str(T('%%{??%s?zero}', 23)), '23')
self.assertEqual(str(T('%%{??%i?zero}', 0)), 'zero')
self.assertEqual(str(T('%%{??1?}%s', '')), '')
self.assertEqual(str(T('%%{??%s?}', 23)), '23')
self.assertEqual(str(T('%%{??0?}%s', '')), '')
self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop')
self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops')
self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)), '1 one')
self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)), '2 not_one')
self.assertEqual(str(T('%%{??on[0]} %i', 1)), ' 1')
self.assertEqual(str(T('%%{??on[0]} %s', 0)), 'on 0')
self.assertEqual(str(T('%%{?on[0]} %s', 1)), ' 1')
self.assertEqual(str(T('%%{?on[0]} %i', 2)), 'on 2')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)), '1 one')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)), '2 or_more')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)), '0 zero')
self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)), '1 one')
self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)), '2 hands')
self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)), '0 ')
self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)), '1 ')
self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)), '2 or_more')
self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)), '0 zero')
self.assertEqual(str(T('%i%%{??nd?[0]}', 1)), '1')
self.assertEqual(str(T('%i%%{??nd?[0]}', 2)), '2nd')
self.assertEqual(str(T('%i%%{??nd?[0]}', 0)), '0')
self.assertEqual(str(T('%i%%{?st?[0]}', 1)), '1st')
self.assertEqual(str(T('%i%%{?st?[0]}', 2)), '2')
self.assertEqual(str(T('%i%%{?st?[0]}', 0)), '0')
# sub_dict testing
self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))), '1 is')
self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))), '2 are')
self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))), 'ARE')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))), '2 ')
self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))), '0 ')
self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(
str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))), '1 one')
self.assertEqual(
str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))), '4 other')
self.assertEqual(
str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))), '0 zero')
self.assertEqual(
str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))), '1 one')
self.assertEqual(
str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))),
'2 two_or_more')
self.assertEqual(
str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))), '0 ')
self.assertEqual(
str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))), '1 ')
self.assertEqual(
str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))),
'2 two_or_more')
self.assertEqual(
str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))),
'0 zero')
self.assertEqual(
str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))), '1 ')
self.assertEqual(
str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))), '0 ')
self.assertEqual(
str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))),
'2 two_or_more')
T.force('it')
self.assertEqual(str(T('Hello World')), 'Salve Mondo')
self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
def test_decode(self):
T = languages.translator(self.langpath, self.http_accept_language)
messages = Messages(T)
messages.update({'email_sent':'Email sent', 'test': "ä"})
self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
def __getitem__(self, index):
return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
def __getslice__(self, begin, end):
return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
def decoder(buffer):
encoding = autoDetectXMLEncoding(buffer)
return to_unicode(buffer, charset=encoding)
def upper_fun(s):
return to_bytes(to_unicode(s).upper())
def cap_fun(s):
return to_bytes(to_unicode(s).capitalize())
def encode(self, *a, **b):
if PY2 and a[0] != 'utf8':
return to_unicode(str(self)).encode(*a, **b)
else:
return str(self)
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
| Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com>
| for Web2py project
Utilities and class for UTF8 strings managing
----------------------------------------------
"""
from __future__ import print_function
from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload
__all__ = ['Utf8']
repr_escape_tab = {}
#FIXME PY3
for i in range(1, 32):
repr_escape_tab[i] = to_unicode("\\" + "x%02x" % i)
repr_escape_tab[7] = u'\\a'
repr_escape_tab[8] = u'\\b'
repr_escape_tab[9] = u'\\t'
repr_escape_tab[10] = u'\\n'
repr_escape_tab[11] = u'\\v'
repr_escape_tab[12] = u'\\f'
repr_escape_tab[13] = u'\\r'
repr_escape_tab[ord('\\')] = u'\\\\'
repr_escape_tab2 = repr_escape_tab.copy()
repr_escape_tab2[ord('\'')] = u"\\'"
def sort_key(s):
"""Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
is used for utf-8 and unicode strings sorting and for utf-8 strings
def sort_function(x):
return to_unicode(x, 'utf-8').lower()
def decoder(buffer):
encoding = autoDetectXMLEncoding(buffer)
return to_unicode(buffer, charset=encoding)
def __len__(self):
return len(to_unicode(self, "utf-8"))
def title_fun(s):
return to_bytes(to_unicode(s).title())
def __getslice__(self, begin, end):
return str.__new__(
Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
def __len__(self):
return len(to_unicode(self, 'utf-8'))
def test_plain(self):
T = languages.translator(self.langpath, self.http_accept_language)
self.assertEqual(str(T('Hello World')),
'Hello World')
self.assertEqual(str(T('Hello World## comment')),
'Hello World')
self.assertEqual(str(T.M('**Hello World**')),
'<strong>Hello World</strong>')
# sub_tuple testing
self.assertEqual(str(T('%s %%{shop}', 1)),
'1 shop')
self.assertEqual(str(T('%s %%{shop}', 2)),
'2 shops')
self.assertEqual(str(T('%%{quark(%s)}', 1)),
'quark')
self.assertEqual(str(T('%%{quark(%i)}', 2)),
'quarks')
self.assertEqual(str(T('%%{!quark(%s)}', 1)),
'Quark')
self.assertEqual(str(T('%%{!!quark(%i)}', 2)),
'Quarks')
self.assertEqual(str(T('%%{!!!quark(%s)}', 0)),
'QUARKS')
self.assertEqual(str(T('%%{?an?%i}', 1)),
'an')
self.assertEqual(str(T('%%{?an?%s}', 0)),
'0')
self.assertEqual(str(T('%%{??%i}', 1)),
'')
self.assertEqual(str(T('%%{??%s}', 2)),
'2')
self.assertEqual(str(T('%%{?%i}', 1)),
'')
self.assertEqual(str(T('%%{?%s}', 0)),
'0')
self.assertEqual(str(T('%%{?one?%i?zero}', 1)),
'one')
self.assertEqual(str(T('%%{?one?%s?zero}', 23)),
'23')
self.assertEqual(str(T('%%{?one?%i?zero}', 0)),
'zero')
self.assertEqual(str(T('%%{?one?%s?}', 1)),
'one')
self.assertEqual(str(T('%%{?one?%i?}', 23)),
'23')
self.assertEqual(str(T('%%{?one?%s?}', 0)),
'')
self.assertEqual(str(T('%%{??%i?zero}', 1)),
'')
self.assertEqual(str(T('%%{??%s?zero}', 23)),
'23')
self.assertEqual(str(T('%%{??%i?zero}', 0)),
'zero')
self.assertEqual(str(T('%%{??1?}%s', '')),
'')
self.assertEqual(str(T('%%{??%s?}', 23)),
'23')
self.assertEqual(str(T('%%{??0?}%s', '')),
'')
self.assertEqual(str(T('%s %%{shop[0]}', 1)),
'1 shop')
self.assertEqual(str(T('%s %%{shop[0]}', 2)),
'2 shops')
self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)),
'1 one')
self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)),
'2 not_one')
self.assertEqual(str(T('%%{??on[0]} %i', 1)),
' 1')
self.assertEqual(str(T('%%{??on[0]} %s', 0)),
'on 0')
self.assertEqual(str(T('%%{?on[0]} %s', 1)),
' 1')
self.assertEqual(str(T('%%{?on[0]} %i', 2)),
'on 2')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)),
'1 one')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)),
'2 or_more')
self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)),
'0 zero')
self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)),
'1 one')
self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)),
'2 hands')
self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)),
'0 ')
self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)),
'1 ')
self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)),
'2 or_more')
self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)),
'0 zero')
self.assertEqual(str(T('%i%%{??nd?[0]}', 1)),
'1')
self.assertEqual(str(T('%i%%{??nd?[0]}', 2)),
'2nd')
self.assertEqual(str(T('%i%%{??nd?[0]}', 0)),
'0')
self.assertEqual(str(T('%i%%{?st?[0]}', 1)),
'1st')
self.assertEqual(str(T('%i%%{?st?[0]}', 2)),
'2')
self.assertEqual(str(T('%i%%{?st?[0]}', 0)),
'0')
# sub_dict testing
self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))),
'1 is')
self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))),
'2 are')
self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))),
'ARE')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))),
'2 ')
self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))),
'0 ')
self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))),
'2 not_one')
self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
'0 not_one')
self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))),
'4 other')
self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))),
'0 zero')
self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))),
'1 one')
self.assertEqual(str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))),
'2 two_or_more')
self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))),
'0 ')
self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))),
'2 two_or_more')
self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))),
'0 zero')
self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))),
'1 ')
self.assertEqual(str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))),
'0 ')
self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))),
'2 two_or_more')
T.force('it')
self.assertEqual(str(T('Hello World')),
'Salve Mondo')
self.assertEqual(to_unicode(T('Hello World')),
'Salve Mondo')
def __getitem__(self, index):
return str.__new__(
Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
def __len__(self):
return len(to_unicode(self, 'utf-8'))
