Пример #1
0
    def __repr__(self):
        r''' # note that we use raw strings to avoid having to use double back slashes below
        NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function::

            utf8.__repr__() works same as str.repr() when processing ascii string
            >>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'"
            True
            >>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\''
            True
            >>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"'
            True
            >>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\''
            True
            >>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
            True

        Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string::

            >>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字')
            True
            >>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字')
            True
            >>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字")
            True
            >>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
            True
            >>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
            True
        '''
        if str.find(self, "'") >= 0 and str.find(self, '"') < 0:  # only single quote exists
            return '"' + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab), 'utf-8') + '"'
        else:
            return "'" + to_native(to_unicode(self, 'utf-8').translate(repr_escape_tab2), 'utf-8') + "'"
Пример #2
0
def sort_key(s):
    """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
    is used for utf-8 and unicode strings sorting and for utf-8 strings
    comparison

    Note:
        pyuca is a very memory cost module! It loads the whole
        "allkey.txt" file (~2mb!) into the memory. But this
        functionality is needed only when sort_key() is called as a
        part of sort() function or when Utf8 strings are compared.

    So, it is a lazy "sort_key" function which (ONLY ONCE, ON ITS
    FIRST CALL) imports pyuca and replaces itself with a real
    sort_key() function
    """
    global sort_key
    try:
        from gluon.contrib.pyuca import unicode_collator
        unicode_sort_key = unicode_collator.sort_key
        sort_key = lambda s: unicode_sort_key(
            to_unicode(s, 'utf-8') if isinstance(s, str) else s)
    except:
        sort_key = lambda s: (
            to_unicode(s, 'utf-8') if isinstance(s, str) else s).lower()
    return sort_key(s)
Пример #3
0
def write_file(file, contents):
	file.write('# -*- coding: utf-8 -*-\n{\n')
	for key in sorted(contents, key = sort_function):
		file.write('%s: %s,\n' % (repr(to_unicode(key)),
								repr(to_unicode(contents[key]))))
	file.write('}\n')
	file.close()
Пример #4
0
def truncate(string, length, dots='...'):
    """Returns string of length < *length* or truncate string with adding
    *dots* suffix to the string's end

    Args:
        length (int): max length of string
        dots (str or unicode): string suffix, when string is cutted

    Returns:
        (utf8-str): original or cutted string
    """
    text = to_unicode(string, 'utf-8')
    dots = to_unicode(dots, 'utf-8') if isinstance(dots, str) else dots
    if len(text) > length:
        text = text[:length - len(dots)] + dots
    return str.__new__(Utf8, text.encode('utf-8'))
Пример #5
0
def chapter():
    try:
        from gluon._compat import to_bytes, to_native, to_unicode
    except ImportError:
        redirect(URL('index', vars=dict(FLASH_MSG = "CompatError")))
    book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0)
    subfolder = get_subfolder(book_id)
    info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE)
    chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE)
    chapter_title = chapters[chapter_id][1]
    response.title = '%s - %s' % (info['title'], to_unicode(chapter_title))
    filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
    dest = os.path.join(request.folder, 'static_chaps', subfolder, '%.2i.html' % chapter_id)
    if not FORCE_RENDER:
        response.headers['Cache-Control'] = 'public, must-revalidate'
        response.headers['Expires'] = calc_date()
        response.headers['Pragma'] = None
    if (not os.path.isfile(dest)) or FORCE_RENDER:
        try:
            content = open(filename, 'rt', encoding='utf-8').read()
        except FileNotFoundError:
            redirect(URL('index', vars=dict(FLASH_MSG = 'FileNotFoundError')))
        content = convert2html(book_id, content).xml()
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        open(dest, 'wb').write(content)
        content = XML(content)
        return locals()
    else:
        content = XML(open(dest, 'rt', encoding='utf-8').read())
        return locals()
Пример #6
0
 def test_plain(self):
     T = languages.translator(self.langpath, self.http_accept_language)
     self.assertEqual(str(T('Hello World')),
                      'Hello World')
     self.assertEqual(str(T('Hello World## comment')),
                      'Hello World')
     self.assertEqual(str(T('%s %%{shop}', 1)),
                      '1 shop')
     self.assertEqual(str(T('%s %%{shop}', 2)),
                      '2 shops')
     self.assertEqual(str(T('%s %%{shop[0]}', 1)),
                      '1 shop')
     self.assertEqual(str(T('%s %%{shop[0]}', 2)),
                      '2 shops')
     self.assertEqual(str(T('%s %%{quark[0]}', 1)),
                      '1 quark')
     self.assertEqual(str(T('%s %%{quark[0]}', 2)),
                      '2 quarks')
     self.assertEqual(str(T.M('**Hello World**')),
                      '<strong>Hello World</strong>')
     T.force('it')
     self.assertEqual(str(T('Hello World')),
                      'Salve Mondo')
     self.assertEqual(to_unicode(T('Hello World')),
                      'Salve Mondo')
Пример #7
0
 def __new__(cls, content='', codepage='utf-8'):
     if isinstance(content, unicodeT):
         return str.__new__(cls, to_native(content, 'utf-8'))
     elif codepage in ('utf-8', 'utf8') or isinstance(content, cls):
         return str.__new__(cls, content)
     else:
         return str.__new__(cls, to_native(to_unicode(content, codepage), 'utf-8'))
Пример #8
0
def ord(char):
    """Returns unicode id for utf8 or unicode *char* character
    SUPPOSE that *char* is an utf-8 or unicode character only
    """
    if isinstance(char, unicodeT):
        return __builtin__.ord(char)
    return __builtin__.ord(to_unicode(char, 'utf-8'))
Пример #9
0
 def __new__(cls, content="", codepage="utf-8"):
     if isinstance(content, unicodeT):
         return str.__new__(cls, to_native(content, "utf-8"))
     elif codepage in ("utf-8", "utf8") or isinstance(content, cls):
         return str.__new__(cls, content)
     else:
         return str.__new__(cls, to_native(to_unicode(content, codepage), "utf-8"))
Пример #10
0
 def __new__(cls, content='', codepage='utf-8'):
     if isinstance(content, unicodeT):
         return str.__new__(cls, to_native(content, 'utf-8'))
     elif codepage in ('utf-8', 'utf8') or isinstance(content, cls):
         return str.__new__(cls, content)
     else:
         return str.__new__(
             cls, to_native(to_unicode(content, codepage), 'utf-8'))
Пример #11
0
def WIKI(text, encoding="utf8", safe_mode='escape', html4tags=False, **attributes):
    if not text:
        test = ''
    if 'extras' in attributes:
        extras = attributes['extras']
        del attributes['extras']
    else:
        extras=None
    text = to_unicode(text, encoding, 'replace')

    return XML(markdown(text,extras=extras,
                        safe_mode=safe_mode, html4tags=html4tags)\
                   .encode(encoding,'xmlcharrefreplace'),**attributes)
Пример #12
0
    def __repr__(self):
        r''' # note that we use raw strings to avoid having to use double back slashes below
        NOTE! This function is a clone of web2py:gluon.languages.utf_repl() function::

            utf8.__repr__() works same as str.repr() when processing ascii string
            >>> repr(Utf8('abc')) == repr(Utf8("abc")) == repr('abc') == repr("abc") == "'abc'"
            True
            >>> repr(Utf8('a"b"c')) == repr('a"b"c') == '\'a"b"c\''
            True
            >>> repr(Utf8("a'b'c")) == repr("a'b'c") == '"a\'b\'c"'
            True
            >>> repr(Utf8('a\'b"c')) == repr('a\'b"c') == repr(Utf8("a'b\"c")) == repr("a'b\"c") == '\'a\\\'b"c\''
            True
            >>> repr(Utf8('a\r\nb')) == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
            True

        Unlike str.repr(), Utf8.__repr__() remains utf8 content when processing utf8 string::

            >>> repr(Utf8('中文字')) == repr(Utf8("中文字")) == "'中文字'" != repr('中文字')
            True
            >>> repr(Utf8('中"文"字')) == "'中\"文\"字'" != repr('中"文"字')
            True
            >>> repr(Utf8("中'文'字")) == '"中\'文\'字"' != repr("中'文'字")
            True
            >>> repr(Utf8('中\'文"字')) == repr(Utf8("中'文\"字")) == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
            True
            >>> repr(Utf8('中\r\n文')) == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
            True
        '''
        if str.find(self, "'") >= 0 and str.find(
                self, '"') < 0:  # only single quote exists
            return '"' + to_native(
                to_unicode(self, 'utf-8').translate(repr_escape_tab),
                'utf-8') + '"'
        else:
            return "'" + to_native(
                to_unicode(self, 'utf-8').translate(repr_escape_tab2),
                'utf-8') + "'"
Пример #13
0
def search():
    def fix_relative_link(match):
        return "%s%s%s%s" % (
            match.group(1), '../chapter/', book_id, match.group(3)
        )  # link rewritten to be relative to the search URL

    try:
        from gluon._compat import to_bytes, to_native, to_unicode
    except ImportError:
        redirect(URL('index', vars=dict(FLASH_MSG="CompatError")))
    book_id = request.args(0) or redirect(URL('index'))
    search = request.vars.search or redirect(URL('chapter', args=book_id))
    subfolder = get_subfolder(book_id)
    info = cache.ram('info_%s' % subfolder,
                     lambda: get_info(subfolder),
                     time_expire=TIME_EXPIRE)
    chapters = cache.ram('chapters_%s' % subfolder,
                         lambda: get_chapters(subfolder),
                         time_expire=TIME_EXPIRE)
    results = []
    content = H2('No results for "%s"' % search)
    relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])')
    for chapter in chapters:
        chapter_id = int(chapter[0])
        filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
        data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '')
        k = data.lower().find(to_unicode(search).lower())
        if k >= 0:
            snippet = data[data.rfind('\n\n', 0, k) +
                           1:data.find('\n\n', k)].strip()
            snippet = relative_link_re.sub(fix_relative_link, snippet)
            results.append((chapter[0], chapter[1], chapter[2],
                            convert2html(book_id, snippet)))
            content = CAT(*[
                DIV(
                    H2(
                        A(chapter[1],
                          _href=URL('chapter',
                                    vars=dict(search=search),
                                    args=(book_id, chapter[0],
                                          chapter[2])))), chapter[3], BR(),
                    A('more',
                      _href=URL('chapter',
                                vars=dict(search=search),
                                args=(book_id, chapter[0], chapter[2])),
                      _class="btn")) for chapter in results
            ])
    response.view = 'default/chapter.html'
    return locals()
Пример #14
0
 def test_plain(self):
     T = languages.translator(self.langpath, self.http_accept_language)
     self.assertEqual(str(T('Hello World')), 'Hello World')
     self.assertEqual(str(T('Hello World## comment')), 'Hello World')
     self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop')
     self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops')
     self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop')
     self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops')
     self.assertEqual(str(T('%s %%{quark[0]}', 1)), '1 quark')
     self.assertEqual(str(T('%s %%{quark[0]}', 2)), '2 quarks')
     self.assertEqual(str(T.M('**Hello World**')),
                      '<strong>Hello World</strong>')
     T.force('it')
     self.assertEqual(str(T('Hello World')), 'Salve Mondo')
     self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
Пример #15
0
def WIKI(text,
         encoding="utf8",
         safe_mode='escape',
         html4tags=False,
         **attributes):
    if not text:
        test = ''
    if 'extras' in attributes:
        extras = attributes['extras']
        del attributes['extras']
    else:
        extras = None
    text = to_unicode(text, encoding, 'replace')

    return XML(markdown(text,extras=extras,
                        safe_mode=safe_mode, html4tags=html4tags)\
                   .encode(encoding,'xmlcharrefreplace'),**attributes)
Пример #16
0
def write_dict(filename, contents):
    if '__corrupted__' in contents:
        return
    fp = None
    try:
        fp = LockedFile(filename, 'w')
        fp.write('# -*- coding: utf-8 -*-\n{\n')
        for key in sorted(contents, key=lambda x: to_unicode(x, 'utf-8').lower()):
            fp.write('%s: %s,\n' % (repr(Utf8(key)),
                                    repr(Utf8(contents[key]))))
        fp.write('}\n')
    except (IOError, OSError):
        if is_writable():
            logging.warning('Unable to write to file %s' % filename)
        return
    finally:
        if fp:
            fp.close()
Пример #17
0
def search():
    def fix_relative_link(match):
        return "%s%s%s%s" % (match.group(1),
                             '../chapter/',
                             book_id,
                             match.group(3))  # link rewritten to be relative to the search URL

    try:
        from gluon._compat import to_bytes, to_native, to_unicode
    except ImportError:
        redirect(URL('index', vars=dict(FLASH_MSG = "CompatError")))
    book_id = request.args(0) or redirect(URL('index'))
    search = request.vars.search or redirect(URL('chapter', args=book_id))
    subfolder = get_subfolder(book_id)
    info = cache.ram('info_%s' % subfolder, lambda: get_info(subfolder), time_expire=TIME_EXPIRE)
    chapters = cache.ram('chapters_%s' % subfolder, lambda: get_chapters(subfolder), time_expire=TIME_EXPIRE)
    results = []
    content = H2('No results for "%s"' % search)
    relative_link_re = re.compile('(\[\[.*)(\.\.)(\/[0-9][0-9](?:#.*)?\]\])')
    for chapter in chapters:
        chapter_id = int(chapter[0])
        filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
        data = open(filename, 'rt', encoding='utf-8').read().replace('\r', '')
        k = data.lower().find(to_unicode(search).lower())
        if k >= 0:
            snippet = data[data.rfind('\n\n', 0, k) + 1:data.find('\n\n', k)].strip()
            snippet = relative_link_re.sub(fix_relative_link, snippet)
            results.append((chapter[0], chapter[1], chapter[2], convert2html(book_id, snippet)))
            content = CAT(*[DIV(H2(A(chapter[1],
                                     _href=URL('chapter',
                                               vars=dict(search=search),
                                               args=(book_id, chapter[0], chapter[2])))),
                                chapter[3], BR(),
                                A('more', _href=URL('chapter',
                                                    vars=dict(search=search),
                                                    args=(book_id, chapter[0], chapter[2])), _class="btn"))
                            for chapter in results])
    response.view = 'default/chapter.html'
    return locals()
Пример #18
0
def chapter():
    try:
        from gluon._compat import to_bytes, to_native, to_unicode
    except ImportError:
        redirect(URL('index', vars=dict(FLASH_MSG="CompatError")))
    book_id, chapter_id = request.args(0), request.args(1, cast=int, default=0)
    subfolder = get_subfolder(book_id)
    info = cache.ram('info_%s' % subfolder,
                     lambda: get_info(subfolder),
                     time_expire=TIME_EXPIRE)
    chapters = cache.ram('chapters_%s' % subfolder,
                         lambda: get_chapters(subfolder),
                         time_expire=TIME_EXPIRE)
    chapter_title = chapters[chapter_id][1]
    response.title = '%s - %s' % (info['title'], to_unicode(chapter_title))
    filename = os.path.join(FOLDER, subfolder, '%.2i.markmin' % chapter_id)
    dest = os.path.join(request.folder, 'static_chaps', subfolder,
                        '%.2i.html' % chapter_id)
    if not FORCE_RENDER:
        response.headers['Cache-Control'] = 'public, must-revalidate'
        response.headers['Expires'] = calc_date()
        response.headers['Pragma'] = None
    if (not os.path.isfile(dest)) or FORCE_RENDER:
        try:
            content = open(filename, 'rt', encoding='utf-8').read()
        except FileNotFoundError:
            redirect(URL('index', vars=dict(FLASH_MSG='FileNotFoundError')))
        content = convert2html(book_id, content).xml()
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        open(dest, 'wb').write(content)
        content = XML(content)
        return locals()
    else:
        content = XML(open(dest, 'rt', encoding='utf-8').read())
        return locals()
Пример #19
0
 def sub_dict(m):
     """ word(key or num)
         !word(key or num), !!word(key or num), !!!word(key or num)
         ?word1?word(key or num)
              ??word(key or num), ?word(key or num)
         ?word1?word?word0(key or num)
         ?word1?word?(key or num)
              ??word?word0(key or num)
         ?word1?word?(key or num)
              ??word?(key or num), ?word?(key or num)
     """
     w, n = m.group('w', 'n')
     c = w[0]
     n = int(n) if n.isdigit() else symbols[n]
     if c not in '!?':
         return self.plural(w, n)
     elif c == '?':
         # ?[word1]?word[?word0](key or num), ?[word1]?word(key or num) or ?word(key or num)
         (p1, sep, p2) = w[1:].partition("?")
         part1 = p1 if sep else ""
         (part2, sep, part3) = (p2 if sep else p1).partition("?")
         if not sep:
             part3 = part2
         num = int(n)
         return part1 if num == 1 else part3 if num == 0 else part2
     elif w.startswith('!!!'):
         word = w[3:]
         fun = upper_fun
     elif w.startswith('!!'):
         word = w[2:]
         fun = title_fun
     else:
         word = w[1:]
         fun = cap_fun
     s = fun(self.plural(word, n))
     return s if PY2 else to_unicode(s)
Пример #20
0
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
| Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com>
| for Web2py project

Utilities and class for UTF8 strings managing
----------------------------------------------
"""
from __future__ import print_function
from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload

__all__ = ['Utf8']

repr_escape_tab = {}
#FIXME PY3
for i in range(1, 32):
    repr_escape_tab[i] = to_unicode("\\"+"x%02x" % i)
repr_escape_tab[7] = u'\\a'
repr_escape_tab[8] = u'\\b'
repr_escape_tab[9] = u'\\t'
repr_escape_tab[10] = u'\\n'
repr_escape_tab[11] = u'\\v'
repr_escape_tab[12] = u'\\f'
repr_escape_tab[13] = u'\\r'
repr_escape_tab[ord('\\')] = u'\\\\'
repr_escape_tab2 = repr_escape_tab.copy()
repr_escape_tab2[ord('\'')] = u"\\'"


def sort_key(s):
    """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
    is used for utf-8 and unicode strings sorting and for utf-8 strings
Пример #21
0
 def test_decode(self):
     T = languages.translator(self.langpath, self.http_accept_language)
     messages = Messages(T)
     messages.update({'email_sent': 'Email sent', 'test': "ä"})
     self.assertEqual(to_unicode(messages.email_sent, 'utf-8'),
                      'Email sent')
Пример #22
0
 def test_plain(self):
     T = languages.translator(self.langpath, self.http_accept_language)
     self.assertEqual(str(T('Hello World')), 'Hello World')
     self.assertEqual(str(T('Hello World## comment')), 'Hello World')
     self.assertEqual(str(T.M('**Hello World**')),
                      '<strong>Hello World</strong>')
     # sub_tuple testing
     self.assertEqual(str(T('%s %%{shop}', 1)), '1 shop')
     self.assertEqual(str(T('%s %%{shop}', 2)), '2 shops')
     self.assertEqual(str(T('%%{quark(%s)}', 1)), 'quark')
     self.assertEqual(str(T('%%{quark(%i)}', 2)), 'quarks')
     self.assertEqual(str(T('%%{!quark(%s)}', 1)), 'Quark')
     self.assertEqual(str(T('%%{!!quark(%i)}', 2)), 'Quarks')
     self.assertEqual(str(T('%%{!!!quark(%s)}', 0)), 'QUARKS')
     self.assertEqual(str(T('%%{?an?%i}', 1)), 'an')
     self.assertEqual(str(T('%%{?an?%s}', 0)), '0')
     self.assertEqual(str(T('%%{??%i}', 1)), '')
     self.assertEqual(str(T('%%{??%s}', 2)), '2')
     self.assertEqual(str(T('%%{?%i}', 1)), '')
     self.assertEqual(str(T('%%{?%s}', 0)), '0')
     self.assertEqual(str(T('%%{?one?%i?zero}', 1)), 'one')
     self.assertEqual(str(T('%%{?one?%s?zero}', 23)), '23')
     self.assertEqual(str(T('%%{?one?%i?zero}', 0)), 'zero')
     self.assertEqual(str(T('%%{?one?%s?}', 1)), 'one')
     self.assertEqual(str(T('%%{?one?%i?}', 23)), '23')
     self.assertEqual(str(T('%%{?one?%s?}', 0)), '')
     self.assertEqual(str(T('%%{??%i?zero}', 1)), '')
     self.assertEqual(str(T('%%{??%s?zero}', 23)), '23')
     self.assertEqual(str(T('%%{??%i?zero}', 0)), 'zero')
     self.assertEqual(str(T('%%{??1?}%s', '')), '')
     self.assertEqual(str(T('%%{??%s?}', 23)), '23')
     self.assertEqual(str(T('%%{??0?}%s', '')), '')
     self.assertEqual(str(T('%s %%{shop[0]}', 1)), '1 shop')
     self.assertEqual(str(T('%s %%{shop[0]}', 2)), '2 shops')
     self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)), '1 one')
     self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)), '2 not_one')
     self.assertEqual(str(T('%%{??on[0]} %i', 1)), ' 1')
     self.assertEqual(str(T('%%{??on[0]} %s', 0)), 'on 0')
     self.assertEqual(str(T('%%{?on[0]} %s', 1)), ' 1')
     self.assertEqual(str(T('%%{?on[0]} %i', 2)), 'on 2')
     self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)), '1 one')
     self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)), '2 or_more')
     self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)), '0 zero')
     self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)), '1 one')
     self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)), '2 hands')
     self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)), '0 ')
     self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)), '1 ')
     self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)), '2 or_more')
     self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)), '0 zero')
     self.assertEqual(str(T('%i%%{??nd?[0]}', 1)), '1')
     self.assertEqual(str(T('%i%%{??nd?[0]}', 2)), '2nd')
     self.assertEqual(str(T('%i%%{??nd?[0]}', 0)), '0')
     self.assertEqual(str(T('%i%%{?st?[0]}', 1)), '1st')
     self.assertEqual(str(T('%i%%{?st?[0]}', 2)), '2')
     self.assertEqual(str(T('%i%%{?st?[0]}', 0)), '0')
     # sub_dict testing
     self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))), '1 is')
     self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))), '2 are')
     self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))), 'ARE')
     self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))),
                      '1 ')
     self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))),
                      '2 not_one')
     self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
                      '0 not_one')
     self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))),
                      '1 one')
     self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))),
                      '2 not_one')
     self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))),
                      '0 not_one')
     self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))),
                      '1 one')
     self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))), '2 ')
     self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))), '0 ')
     self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))),
                      '1 ')
     self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))),
                      '2 not_one')
     self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))),
                      '1 ')
     self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
                      '0 not_one')
     self.assertEqual(
         str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))), '1 one')
     self.assertEqual(
         str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))), '4 other')
     self.assertEqual(
         str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))), '0 zero')
     self.assertEqual(
         str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))), '1 one')
     self.assertEqual(
         str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))),
         '2 two_or_more')
     self.assertEqual(
         str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))), '0 ')
     self.assertEqual(
         str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))), '1 ')
     self.assertEqual(
         str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))),
         '2 two_or_more')
     self.assertEqual(
         str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))),
         '0 zero')
     self.assertEqual(
         str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))), '1 ')
     self.assertEqual(
         str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))), '0 ')
     self.assertEqual(
         str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))),
         '2 two_or_more')
     T.force('it')
     self.assertEqual(str(T('Hello World')), 'Salve Mondo')
     self.assertEqual(to_unicode(T('Hello World')), 'Salve Mondo')
Пример #23
0
 def test_decode(self):
     T = languages.translator(self.langpath, self.http_accept_language)
     messages = Messages(T)
     messages.update({'email_sent':'Email sent', 'test': "ä"})
     self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
Пример #24
0
 def __getitem__(self, index):
     return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
Пример #25
0
 def __getslice__(self, begin, end):
     return str.__new__(Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
Пример #26
0
def decoder(buffer):
    encoding = autoDetectXMLEncoding(buffer)
    return to_unicode(buffer, charset=encoding)
Пример #27
0
def upper_fun(s):
    return to_bytes(to_unicode(s).upper())
Пример #28
0
def cap_fun(s):
    return to_bytes(to_unicode(s).capitalize())
Пример #29
0
 def encode(self, *a, **b):
     if PY2 and a[0] != 'utf8':
         return to_unicode(str(self)).encode(*a, **b)
     else:
         return str(self)
Пример #30
0
| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
| Created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop©gmail.com>
| for Web2py project

Utilities and class for UTF8 strings managing
----------------------------------------------
"""
from __future__ import print_function
from gluon._compat import builtin as __builtin__, unicodeT, iteritems, to_unicode, to_native, reload

__all__ = ['Utf8']

repr_escape_tab = {}
#FIXME PY3
for i in range(1, 32):
    repr_escape_tab[i] = to_unicode("\\" + "x%02x" % i)
repr_escape_tab[7] = u'\\a'
repr_escape_tab[8] = u'\\b'
repr_escape_tab[9] = u'\\t'
repr_escape_tab[10] = u'\\n'
repr_escape_tab[11] = u'\\v'
repr_escape_tab[12] = u'\\f'
repr_escape_tab[13] = u'\\r'
repr_escape_tab[ord('\\')] = u'\\\\'
repr_escape_tab2 = repr_escape_tab.copy()
repr_escape_tab2[ord('\'')] = u"\\'"


def sort_key(s):
    """Unicode Collation Algorithm (UCA) (http://www.unicode.org/reports/tr10/)
    is used for utf-8 and unicode strings sorting and for utf-8 strings
Пример #31
0
def sort_function(x):
    return to_unicode(x, 'utf-8').lower()
Пример #32
0
def decoder(buffer):
    encoding = autoDetectXMLEncoding(buffer)
    return to_unicode(buffer, charset=encoding)
Пример #33
0
 def __len__(self):
     return len(to_unicode(self, "utf-8"))
Пример #34
0
def title_fun(s):
    return to_bytes(to_unicode(s).title())
Пример #35
0
 def __getslice__(self, begin, end):
     return str.__new__(
         Utf8, to_native(to_unicode(self, 'utf-8')[begin:end], 'utf-8'))
Пример #36
0
 def __len__(self):
     return len(to_unicode(self, 'utf-8'))
Пример #37
0
    def test_plain(self):
        T = languages.translator(self.langpath, self.http_accept_language)
        self.assertEqual(str(T('Hello World')),
                         'Hello World')
        self.assertEqual(str(T('Hello World## comment')),
                         'Hello World')
        self.assertEqual(str(T.M('**Hello World**')),
                         '<strong>Hello World</strong>')
		# sub_tuple testing
        self.assertEqual(str(T('%s %%{shop}', 1)),
                         '1 shop')
        self.assertEqual(str(T('%s %%{shop}', 2)),
                         '2 shops')
        self.assertEqual(str(T('%%{quark(%s)}', 1)),
                         'quark')
        self.assertEqual(str(T('%%{quark(%i)}', 2)),
                         'quarks')
        self.assertEqual(str(T('%%{!quark(%s)}', 1)),
                         'Quark')
        self.assertEqual(str(T('%%{!!quark(%i)}', 2)),
                         'Quarks')
        self.assertEqual(str(T('%%{!!!quark(%s)}', 0)),
                         'QUARKS')
        self.assertEqual(str(T('%%{?an?%i}', 1)),
                         'an')
        self.assertEqual(str(T('%%{?an?%s}', 0)),
                         '0')
        self.assertEqual(str(T('%%{??%i}', 1)),
                         '')
        self.assertEqual(str(T('%%{??%s}', 2)),
                         '2')
        self.assertEqual(str(T('%%{?%i}', 1)),
                         '')
        self.assertEqual(str(T('%%{?%s}', 0)),
                         '0')
        self.assertEqual(str(T('%%{?one?%i?zero}', 1)),
                         'one')
        self.assertEqual(str(T('%%{?one?%s?zero}', 23)),
                         '23')
        self.assertEqual(str(T('%%{?one?%i?zero}', 0)),
                         'zero')
        self.assertEqual(str(T('%%{?one?%s?}', 1)),
                         'one')
        self.assertEqual(str(T('%%{?one?%i?}', 23)),
                         '23')
        self.assertEqual(str(T('%%{?one?%s?}', 0)),
                         '')
        self.assertEqual(str(T('%%{??%i?zero}', 1)),
                         '')
        self.assertEqual(str(T('%%{??%s?zero}', 23)),
                         '23')
        self.assertEqual(str(T('%%{??%i?zero}', 0)),
                         'zero')
        self.assertEqual(str(T('%%{??1?}%s', '')),
                         '')
        self.assertEqual(str(T('%%{??%s?}', 23)),
                         '23')
        self.assertEqual(str(T('%%{??0?}%s', '')),
                         '')
        self.assertEqual(str(T('%s %%{shop[0]}', 1)),
                         '1 shop')
        self.assertEqual(str(T('%s %%{shop[0]}', 2)),
                         '2 shops')
        self.assertEqual(str(T('%i %%{?one?not_one[0]}', 1)),
                         '1 one')
        self.assertEqual(str(T('%i %%{?one?not_one[0]}', 2)),
                         '2 not_one')
        self.assertEqual(str(T('%%{??on[0]} %i', 1)),
                         ' 1')
        self.assertEqual(str(T('%%{??on[0]} %s', 0)),
                         'on 0')
        self.assertEqual(str(T('%%{?on[0]} %s', 1)),
                         ' 1')
        self.assertEqual(str(T('%%{?on[0]} %i', 2)),
                         'on 2')
        self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 1)),
                         '1 one')
        self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 2)),
                         '2 or_more')
        self.assertEqual(str(T('%i %%{?one?or_more?zero[0]}', 0)),
                         '0 zero')
        self.assertEqual(str(T('%i %%{?one?hands?[0]}', 1)),
                         '1 one')
        self.assertEqual(str(T('%s %%{?one?hands?[0]}', 2)),
                         '2 hands')
        self.assertEqual(str(T('%i %%{?one?hands?[0]}', 0)),
                         '0 ')
        self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 1)),
                         '1 ')
        self.assertEqual(str(T('%i %%{??or_more?zero[0]}', 2)),
                         '2 or_more')
        self.assertEqual(str(T('%s %%{??or_more?zero[0]}', 0)),
                         '0 zero')
        self.assertEqual(str(T('%i%%{??nd?[0]}', 1)),
                         '1')
        self.assertEqual(str(T('%i%%{??nd?[0]}', 2)),
                         '2nd')
        self.assertEqual(str(T('%i%%{??nd?[0]}', 0)),
                         '0')
        self.assertEqual(str(T('%i%%{?st?[0]}', 1)),
                         '1st')
        self.assertEqual(str(T('%i%%{?st?[0]}', 2)),
                         '2')
        self.assertEqual(str(T('%i%%{?st?[0]}', 0)),
                         '0')
		# sub_dict testing
        self.assertEqual(str(T('%(key)s %%{is(key)}', dict(key=1))),
                         '1 is')
        self.assertEqual(str(T('%(key)i %%{is(key)}', dict(key=2))),
                         '2 are')
        self.assertEqual(str(T('%%{!!!is(%(key)s)}', dict(key=2))),
                         'ARE')
        self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=1))),
                         '1 ')
        self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=2))),
                         '2 not_one')
        self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
                         '0 not_one')
        self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=1))),
                         '1 one')
        self.assertEqual(str(T('%(key)i %%{?one?not_one(key)}', dict(key=2))),
                         '2 not_one')
        self.assertEqual(str(T('%(key)s %%{?one?not_one(key)}', dict(key=0))),
                         '0 not_one')
        self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=1))),
                         '1 one')
        self.assertEqual(str(T('%(key)s %%{?one?(key)}', dict(key=2))),
                         '2 ')
        self.assertEqual(str(T('%(key)i %%{?one?(key)}', dict(key=0))),
                         '0 ')
        self.assertEqual(str(T('%(key)s %%{??not_one(key)}', dict(key=1))),
                         '1 ')
        self.assertEqual(str(T('%(key)i %%{??not_one(key)}', dict(key=2))),
                         '2 not_one')
        self.assertEqual(str(T('%(key)s %%{?not_one(key)}', dict(key=1))),
                         '1 ')
        self.assertEqual(str(T('%(key)i %%{?not_one(key)}', dict(key=0))),
                         '0 not_one')
        self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=1))),
                         '1 one')
        self.assertEqual(str(T('%(key)i %%{?one?other?zero(key)}', dict(key=4))),
                         '4 other')
        self.assertEqual(str(T('%(key)s %%{?one?other?zero(key)}', dict(key=0))),
                         '0 zero')
        self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=1))),
                         '1 one')
        self.assertEqual(str(T('%(key)s %%{?one?two_or_more?(key)}', dict(key=2))),
                         '2 two_or_more')
        self.assertEqual(str(T('%(key)i %%{?one?two_or_more?(key)}', dict(key=0))),
                         '0 ')
        self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=1))),
                         '1 ')
        self.assertEqual(str(T('%(key)i %%{??two_or_more?zero(key)}', dict(key=2))),
                         '2 two_or_more')
        self.assertEqual(str(T('%(key)s %%{??two_or_more?zero(key)}', dict(key=0))),
                         '0 zero')
        self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=1))),
                         '1 ')
        self.assertEqual(str(T('%(key)s %%{??two_or_more?(key)}', dict(key=0))),
                         '0 ')
        self.assertEqual(str(T('%(key)i %%{??two_or_more?(key)}', dict(key=2))),
                         '2 two_or_more')
        T.force('it')
        self.assertEqual(str(T('Hello World')),
                         'Salve Mondo')
        self.assertEqual(to_unicode(T('Hello World')),
                         'Salve Mondo')
Пример #38
0
 def __getitem__(self, index):
     return str.__new__(
         Utf8, to_native(to_unicode(self, 'utf-8')[index], 'utf-8'))
Пример #39
0
 def __len__(self):
     return len(to_unicode(self, 'utf-8'))