Example #1
0
 def _on_commands(self, line):
     try:
         if self.__state == self.COMMAND:
             line = to_unicode(line)[:-2]  # Remove delimiter '\r\n'
             if not line.strip():
                 raise errors.UnrecognisedCommand()
             i = line.find(' ')
             if i < 0:
                 command = line
                 arg = None
             else:
                 command = line[:i]
                 arg = line[i + 1:].strip()
             method = getattr(self, 'command_' + command.lower(), None)
             if not method:
                 raise errors.NotImplementedCommand(command)
             method(arg)
         elif self.__state == self.DATA:
             line = to_unicode(line)[:-5]  # Remove delimiter '\r\n.\r\n'
             data = []
             for text in line.split(CRLF):
                 if text and text[0] == '.':
                     data.append(text[1:])
                 else:
                     data.append(text)
             self._on_data('\n'.join(data))
         else:
             raise errors.InternalConfusion()
     except Exception as e:
         self._handle_request_exception(e)
Example #2
0
 def get(self):
     self.write(u"抓取中<a href='/'>回首页</a>")
     self.finish()
     feeds = self.feed_list
     #pp = pprint.PrettyPrinter(indent=4)
     for f in feeds:
         d = feedparser.parse(f.feed_url)
         for e in d.entries:
             #pp.pprint(e.content[0].value)
             exist =\
             self.db.query(Entry).filter_by(entry_title=\
             to_unicode(e.title)).first()
             if exist:
                 continue
             entry = Entry(entry_title=to_unicode(e.title))
             try:
                 entry.entry_author = to_unicode(e.author)
             except:
                 entry.entry_author = 'admin'
             entry.entry_link = to_unicode(e.link)
             entry_content = to_unicode(e.content[0].value)
             entry.entry_content = entry_content
             entry.entry_parrent = f.id
             entry.entry_pubdate =\
             datetime.fromtimestamp(mktime(e.updated_parsed))
             self.db.add(entry)
             self.db.commit()
Example #3
0
def load_json_translations(directory):
    """Loads translations from JSON files in a directory."""
    tornado_locale._translations = {}
    for path in os.listdir(directory):
        if not path.endswith(".json"):
            continue
        locale, _ = path.split(".")
        locale_parts = locale.replace('-', '_').split('_')
        locale = "_".join(
            locale_parts[:1] + [part.upper() for part in locale_parts[1:]])
        if not re.match("[a-z]+(_[A-Z]+)?$", locale):
            # Unrecognized locale
            continue
        full_path = os.path.join(directory, path)
        try:
            with open(full_path, "r") as fd:
                all_translations = json.load(fd)
        except TypeError:
            # Invalid translation
            continue
        tornado_locale._translations[locale] = {}
        for key, value in all_translations.iteritems():
            tornado_locale._translations[locale].setdefault("unknown", {})[
                escape.to_unicode(key)] = escape.to_unicode(value)
    tornado_locale._supported_locales = frozenset(
        list(tornado_locale._translations.keys()) +
        [tornado_locale._default_locale])
        def handle_bulk(self, records_list, stages=None, status_code=None, exception=None, uri=None, method=None, **kwargs):
            if len(records_list) > 0:
                first_record = records_list[0]
            else:
                return

            record_for_gelf = copy.deepcopy(first_record)
            record_for_gelf.message = u"{0} {1} {2} \n".format(record_for_gelf.asctime, record_for_gelf.levelname,
                                                               to_unicode(record_for_gelf.message))
            record_for_gelf.short = u"{0} {1} {2}".format(method, to_unicode(uri), status_code)
            record_for_gelf.exc_info = exception
            record_for_gelf.levelno = 20

            for record in records_list[1:]:
                if record.levelno > record_for_gelf.levelno:
                    record_for_gelf.levelno = record.levelno
                    record_for_gelf.lineno = record.lineno
                    record_for_gelf.short = record.message
                if record.exc_info is not None:
                    record_for_gelf.exc_info = traceback.format_exc(record.exc_info)
                    record_for_gelf.short += "\n" + traceback.format_exc(record.exc_info)
                record_for_gelf.message += u" {0} {1} {2} \n".format(record.asctime, record.levelname,
                                                                     to_unicode(record.message))
            if stages is not None:
                for stage_name, stage_start, stage_delta in stages:
                    setattr(record_for_gelf, stage_name + "_stage", str(int(stage_delta*1000)))

            record_for_gelf.name = record_for_gelf.handler
            record_for_gelf.code = status_code
            GELFHandler.handle(self, record_for_gelf)
            self.close()
Example #5
0
 def assertNotIn(self, observed, expected, *args, **kwargs):
     return super(NBViewerTestCase, self).assertNotIn(
         to_unicode(observed),
         to_unicode(expected),
         *args,
         **kwargs
     )
Example #6
0
        def job_callback(future):
            if future.exception() is not None:
                self.log.error('failed applying template %s', self.template_filename)

                exception = future.exception()
                if isinstance(exception, jinja2.TemplateSyntaxError):
                    self.log.error(
                        u'%s in file "%s", line %d\n\t%s',
                        exception.__class__.__name__, to_unicode(exception.filename),
                        exception.lineno, to_unicode(exception.message)
                    )
                elif isinstance(exception, jinja2.TemplateError):
                    self.log.error(u'%s error\n\t%s', exception.__class__.__name__, to_unicode(exception.message))

                if isinstance(future, TracebackFuture):
                    raise_exc_info(future.exc_info())
                else:
                    raise exception

            start_time, result = future.result()

            self.log.stage_tag('tpl')
            self.log.info('applied template %s in %.2fms', self.template_filename, (time.time() - start_time) * 1000)

            callback(utf8(result))
Example #7
0
def send_mail(recepients=None, subject=None, template=None,  template_data=None):
	connection_details = email_model.get_email_details()

	port = int(connection_details['port'])
	security = connection_details['security'] if connection_details['security'] != 'None' else None

	mailer = ThreadedMailer(SMTPMailer(connection_details['address'], port, 
					username = connection_details['username'],
					password = connection_details['password'],
					transport_args = {'security': security},
					log_file = '/var/log/amonone/amonone-mailer.log',
					log_messages=False))


	EMAIL_ROOT = abspath(dirname(__file__))
	TEMPLATES_DIR =  join(EMAIL_ROOT, 'templates')

	env = Environment(loader=FileSystemLoader(TEMPLATES_DIR))
	env.filters['date_local'] = dateformat_local

	template_file = env.get_template("{0}.html".format(template))
	rendered_template = template_file.render(template_data=template_data)

	message = escape.to_unicode(rendered_template)

	subject = escape.to_unicode(subject)

	email_recepients = [x['email'] for x in recepients]

	try: 
		mailer.send_html(connection_details['from_'], email_recepients, subject, message)
	except Exception, e:
		print e
		raise e 
Example #8
0
    async def _finish_with_template(self):
        if not self.environment:
            raise Exception('Cannot apply template, no Jinja2 environment configured')

        if self.handler._headers.get('Content-Type') is None:
            self.handler.set_header('Content-Type', media_types.TEXT_HTML)

        try:
            render_result = await self._render_template_stream_on_ioloop(options.jinja_streaming_render_timeout_ms)
            if self.handler.is_finished():
                return None

            start_time, result = render_result

            self.handler.stages_logger.commit_stage('tpl')
            self.log.info('applied template %s in %.2fms', self.template_filename, (time.time() - start_time) * 1000)

            return result

        except Exception as e:
            self.log.error('failed applying template %s', self.template_filename)

            if isinstance(e, jinja2.TemplateSyntaxError):
                self.log.error(
                    '%s in file "%s", line %d\n\t%s',
                    e.__class__.__name__, to_unicode(e.filename), e.lineno, to_unicode(e.message)
                )
            elif isinstance(e, jinja2.TemplateError):
                self.log.error('%s error\n\t%s', e.__class__.__name__, to_unicode(e.message))

            raise e
Example #9
0
def _exception_to_xml(exc_info, log=debug_log):
    exc_node = etree.Element('exception')

    try:
        trace_node = etree.Element('trace')
        trace = exc_info[2]
        while trace:
            frame = trace.tb_frame
            trace_step_node = etree.Element('step')
            trace_lines = etree.Element('lines')

            try:
                lines, starting_line = inspect.getsourcelines(frame)
            except IOError:
                lines, starting_line = [], None

            for i, l in enumerate(lines):
                line_node = etree.Element('line')
                line_node.append(E.text(to_unicode(l)))
                line_node.append(E.number(str(starting_line + i)))
                if starting_line + i == frame.f_lineno:
                    line_node.set('selected', 'true')
                trace_lines.append(line_node)

            trace_step_node.append(trace_lines)
            trace_step_node.append(E.file(to_unicode(inspect.getfile(frame))))
            trace_step_node.append(E.locals(pprint.pformat(frame.f_locals)))
            trace_node.append(trace_step_node)
            trace = trace.tb_next
        exc_node.append(trace_node)
    except Exception:
        log.exception('cannot add traceback lines')

    exc_node.append(E.text(''.join(map(to_unicode, traceback.format_exception(*exc_info)))))
    return exc_node
Example #10
0
    def handle(self, record):
        entry_attrs = {}
        for field in self.FIELDS:
            val = getattr(record, field)
            if val is not None:
                entry_attrs[field] = str(val)

        entry_attrs['msg'] = to_unicode(record.getMessage())

        try:
            entry = etree.Element("entry", **entry_attrs)
        except ValueError:
            debug_log.exception('error creating log entry with attrs: %s', entry_attrs)
            entry = etree.Element("entry")

        entry.set("asctime", str(datetime.fromtimestamp(record.created)))

        if record.exc_info is not None:
            entry.append(_exception_to_xml(record.exc_info))

        if getattr(record, '_labels', None) is not None:
            labels = E.labels()
            for label in record._labels:
                labels.append(E.label(label))
            entry.append(labels)

        if getattr(record, '_response', None) is not None:
            entry.append(response_to_xml(record._response))

        if getattr(record, '_request', None) is not None:
            entry.append(request_to_xml(record._request))

        if getattr(record, '_debug_response', None) is not None:
            entry.append(E.debug(record._debug_response))

        if getattr(record, '_xslt_profile', None) is not None:
            entry.append(record._xslt_profile)

        if getattr(record, '_xml', None) is not None:
            entry.append(E.text(etree.tostring(record._xml, encoding=unicode)))

        if getattr(record, '_protobuf', None) is not None:
            if isinstance(record._protobuf, basestring):
                entry.append(E.text(record._protobuf))
            else:
                entry.append(E.text(unicode(record._protobuf)))

        if getattr(record, '_text', None) is not None:
            entry.append(E.text(to_unicode(record._text)))

        if getattr(record, '_stage', None) is not None:
            entry.append(E.stage(
                E.name(record._stage.name),
                E.delta(_format_number(record._stage.delta)),
                E.start_delta(_format_number(record._stage.start_delta))
            ))

        self.log_data.append(entry)
Example #11
0
def unicode_encoded_dict(in_dict):
    """
    使用 unicode 重新编码字典
    :param in_dict:
    :return:
    """
    out_dict = {}
    for k, v in iteritems(in_dict):
        out_dict[to_unicode(k)] = to_unicode(v)
    return out_dict
    def get_filters_values_key(self, project, metric_name, f):
        """Get key for filter values list

        :param project:
        :param metric_name:
        :param f: filter name
        """
        return u"{0}-metrics-filter-values:{1}:{2}".format(project,
                                                           to_unicode(metric_name),
                                                           to_unicode(f))
def _params_to_xml(url, logger=debug_log):
    params = etree.Element('params')
    query = frontik.util.get_query_parameters(url)
    for name, values in query.iteritems():
        for value in values:
            try:
                params.append(E.param(to_unicode(value), name=to_unicode(name)))
            except UnicodeDecodeError:
                logger.exception('cannot decode parameter name or value')
                params.append(E.param(repr(value), name=repr(name)))
    return params
Example #14
0
File: util.py Project: hhru/frontik
def make_url(base, **query_args):
    """
    Builds URL from base part and query arguments passed as kwargs.
    Returns unicode string
    """
    qs = make_qs(query_args)

    if qs:
        return to_unicode(base) + ('&' if '?' in base else '?') + qs
    else:
        return to_unicode(base)
Example #15
0
    def _produce_one(self, record):
        entry_attrs = {}
        for field in self.FIELDS:
            val = getattr(record, field)
            if val is not None:
                entry_attrs[field] = to_unicode(str(val))

        entry_attrs['msg'] = to_unicode(record.getMessage())

        try:
            entry = etree.Element('entry', **entry_attrs)
        except ValueError:
            debug_log.exception('error creating log entry with attrs: %s', entry_attrs)
            entry = etree.Element('entry')

        entry.set('asctime', str(datetime.fromtimestamp(record.created)))

        if record.exc_info is not None:
            entry.append(_exception_to_xml(record.exc_info))

        if getattr(record, '_response', None) is not None:
            entry.append(response_to_xml(record._response))

        if getattr(record, '_request', None) is not None:
            entry.append(request_to_xml(record._request))

        if getattr(record, '_balanced_request', None) is not None:
            entry.append(balanced_request_to_xml(record._balanced_request, record._request_retry,
                                                 record._rack, record._datacenter))

        if getattr(record, '_debug_response', None) is not None:
            entry.append(E.debug(record._debug_response))

        if getattr(record, '_xslt_profile', None) is not None:
            entry.append(record._xslt_profile)

        if getattr(record, '_xml', None) is not None:
            entry.append(E.text(etree.tostring(record._xml, encoding='unicode')))

        if getattr(record, '_protobuf', None) is not None:
            entry.append(E.text(str(record._protobuf)))

        if getattr(record, '_text', None) is not None:
            entry.append(E.text(to_unicode(record._text)))

        if getattr(record, '_stage', None) is not None:
            entry.append(E.stage(
                E.name(record._stage.name),
                E.delta(_format_number(record._stage.delta)),
                E.start_delta(_format_number(record._stage.start_delta))
            ))

        return entry
Example #16
0
    def process_formdata(self, valuelist):
        """
        Process data received over the wire from a form.

        This will be called during form construction with data supplied
        through the `formdata` argument.

        :param valuelist: A list of strings to process.
        """
        if valuelist:
            self.data = to_unicode(valuelist[0])
        else:
            self.data = to_unicode(None)
Example #17
0
def _flatten_arguments(args):
    """
    去除请求中单值参数的数组结构

    """
    flattened = {}
    for key in args:
        if len(args[key]) == 1:
            flattened[key] = to_unicode(args[key][0])
        else:
            flattened[key] = [to_unicode(arg) for arg in args[key]]

    return flattened
Example #18
0
 def test_parse_xml(self):
     fd = open(os.path.join(os.path.dirname(__file__), 'data', 'simple.xml'), 'r')
     tree = parse(fd)
     self.assertEqual(tree.getroot().tag, 'data')
     convert = tostring(tree.getroot(), pretty_print=True, xml_declaration=True, encoding='UTF-8')
     # replace any possible conversion differences that are ok
     # Python 3+ native etree does not include xml declaration so we should remove it everywhere
     converted = to_unicode(convert).replace('\n', '').replace(' ', '').replace('\'', '"').\
         replace('<?xmlversion="1.0"encoding="UTF-8"?>', '').strip()
     fd.seek(0)
     base = to_unicode(fd.read()).replace('\n', '').replace(' ', '').\
         replace('<?xmlversion="1.0"encoding="UTF-8"?>', '').strip()
     self.assertEqual(converted, base)
     fd.close()
Example #19
0
def worker_is_alive(port, config):
    try:
        path_beginning, _, path_ending = options.pidfile_template.partition('%(port)s')
        pidfile_pattern = '{}([0-9]+){}'.format(re.escape(path_beginning), re.escape(path_ending))
        pidfile_grep_result = to_unicode(subprocess.check_output(['pgrep', '-f', pidfile_pattern])).strip().split('\n')

        for pid in pidfile_grep_result:
            cmdline = to_unicode(subprocess.check_output(['ps', '-p', pid, '-o', 'command='])).strip()

            if cmdline is not None and str(port) in cmdline and config in cmdline and 'python' in cmdline:
                return True

        return False
    except (IOError, subprocess.CalledProcessError):
        return False
Example #20
0
def ForceUnicode(value):
  """Returns true if the value contains only GSM chars, but also contains at least one
  problematic GSM char, such as a Greek capital letter. In this case, the caller should
  force the UCS-2 SMS encoding so that GSM will not be attempted.
  """
  value = escape.to_unicode(value)
  return _force_unicode_re.search(value) and not _gsm_re.search(value)
Example #21
0
 def __init__(self, template_string, name="<string>", loader=None,
              compress_whitespace=None, autoescape=_UNSET):
     self.name = name
     if compress_whitespace is None:
         compress_whitespace = name.endswith(".html") or \
             name.endswith(".js")
     if autoescape is not _UNSET:
         self.autoescape = autoescape
     elif loader:
         self.autoescape = loader.autoescape
     else:
         self.autoescape = _DEFAULT_AUTOESCAPE
     self.namespace = loader.namespace if loader else {}
     reader = _TemplateReader(name, escape.native_str(template_string))
     self.file = _File(_parse(reader, self))
     self.code = self._generate_python(loader, compress_whitespace)
     try:
         self.compiled = compile(escape.to_unicode(self.code),
                                 "<template %s>" % self.name,
                                 "exec")
     except Exception, e:
         formatted_code = _format_code(self.code).rstrip()
         logging.error("%s code:\n%s", self.name, formatted_code)
         e.error_msg = "%s code:\n%s" % (self.name, formatted_code)
         raise e
Example #22
0
 def __init__(self, template_string, name="<string>", loader=None,
              compress_whitespace=None, autoescape=_UNSET):
     self.name = name
     if compress_whitespace is None:
         compress_whitespace = name.endswith(".html") or \
             name.endswith(".js")
     if autoescape is not _UNSET:
         self.autoescape = autoescape
     elif loader:
         self.autoescape = loader.autoescape
     else:
         self.autoescape = _DEFAULT_AUTOESCAPE
     self.namespace = loader.namespace if loader else {}
     reader = _TemplateReader(name, escape.native_str(template_string))
     self.file = _File(self, _parse(reader, self))
     self.code = self._generate_python(loader, compress_whitespace)
     self.loader = loader
     try:
         # Under python2.5, the fake filename used here must match
         # the module name used in __name__ below.
         self.compiled = compile(
             escape.to_unicode(self.code),
             "%s.generated.py" % self.name.replace('.', '_'),
             "exec")
     except Exception:
         formatted_code = _format_code(self.code).rstrip()
         logging.error("%s code:\n%s", self.name, formatted_code)
         raise
Example #23
0
    def _handle_message(self, opcode, data):
        if self.client_terminated:
            return

        if opcode == 0x1:
            # UTF-8 data
            try:
                decoded = data.decode("utf-8")
            except UnicodeDecodeError:
                self._abort()
                return
            self.async_callback(self.handler.on_message)(decoded)
        elif opcode == 0x2:
            # Binary data
            self.async_callback(self.handler.on_message)(data)
        elif opcode == 0x8:
            # Close
            self.client_terminated = True
            if len(data) >= 2:
                self.handler.close_code = struct.unpack('>H', data[:2])[0]
            if len(data) > 2:
                self.handler.close_reason = to_unicode(data[2:])
            self.close()
        elif opcode == 0x9:
            # Ping
            self._write_frame(True, 0xA, data)
        elif opcode == 0xA:
            # Pong
            self.async_callback(self.handler.on_pong)(data)
        else:
            self._abort()
Example #24
0
 def decode_argument(self, value, name=None):
     assert type(value) == bytes_type, repr(value)
     # use self.request.arguments directly to avoid recursion
     if 'encoding' in self.request.arguments:
         return value.decode(to_unicode(self.request.arguments['encoding'][0]))
     else:
         return value
Example #25
0
 def get_following_users(self, user):
     doc = yield self._db.followers.find_one({"user": user}, {"_id":0, "following":1})
     if doc and "following" in doc:
         ret = yield [self.find_user(_, True) for _ in doc["following"] if _]
     else:
         ret = []
     raise gen.Return(sorted(ret, key=lambda x: pinyin(to_unicode(("real_name" in x and x["real_name"]) or ""), style=TONE2)))
Example #26
0
    def get_notebook_argument(self, argname):
        # Assuming a request on the form "{'argname':arg}"
        body = json.loads(escape.to_unicode(self.request.body))
        arg = body[argname]

        # Currently assuming arg is a filename relative to
        # where the server was started from, later we may
        # want to accept urls or full notebooks as well.
        if not isinstance(arg, string_types):
            raise web.HTTPError(400, "Expecting a filename.")

        # Check that file exists
        path = os.path.join(self.params["cwd"], arg)
        if not os.path.exists(path):
            # Assume file is URI
            r = requests.get(arg)

        # Let nbformat do the reading and validation
        try:
            if os.path.exists(path):
                nb = nbformat.read(path, as_version=4)
            else:
                nb = nbformat.reads(r.text, as_version=4)
        except:
            raise web.HTTPError(400, "Invalid notebook: %s" % truncate_filename(arg))

        return nb
Example #27
0
 def Run(self, query_str):
   """Runs the parser on the provided query expression and returns
   the resulting query tree.
   """
   # Convert query_str to Unicode since picoparser expects Unicode for non-ASCII characters.
   query_tree, _ = run_text_parser(self._expr_parser, escape.to_unicode(query_str))
   return query_tree
Example #28
0
 def __init__(self, template_string, name="<string>", loader=None, compress_whitespace=None, autoescape=_UNSET):
     self.name = name
     if compress_whitespace is None:
         compress_whitespace = name.endswith(".html") or name.endswith(".js")
     if autoescape is not _UNSET:
         self.autoescape = autoescape
     elif loader:
         self.autoescape = loader.autoescape
     else:
         self.autoescape = _DEFAULT_AUTOESCAPE
     self.namespace = loader.namespace if loader else {}
     reader = _TemplateReader(name, escape.native_str(template_string))
     self.file = _File(self, _parse(reader, self))
     self.code = self._generate_python(loader, compress_whitespace)
     self.loader = loader
     try:
         # Under python2.5, the fake filename used here must match
         # the module name used in __name__ below.
         # The dont_inherit flag prevents template.py's future imports
         # from being applied to the generated code.
         self.compiled = compile(
             escape.to_unicode(self.code), "%s.generated.py" % self.name.replace(".", "_"), "exec", dont_inherit=True
         )
     except Exception:
         formatted_code = _format_code(self.code).rstrip()
         app_log.error("%s code:\n%s", self.name, formatted_code)
         raise
    def get_filters_names_key(self, project, metric_name):
        """Get key for metrics filters names

        :param project: project name
        :param metric_name: metric name
        """
        return u"{0}-metrics-filters:{1}".format(project, to_unicode(metric_name))
Example #30
0
 def setUp(self):
     self.hostname = 'client'
     self.mail = '*****@*****.**'
     self.rcpt = ['*****@*****.**', '*****@*****.**']
     self.data = 'This is a message.'
     self.message = email.message_from_string(to_unicode(self.data))
     super(SMTPRequestTest, self).setUp()
 def delete_callback(self, data, response):
     self.json.put({'delete': to_unicode(data)})
Example #32
0
 def __call__(self, field, **kwargs):
     kwargs.setdefault('id', field.id)
     return '<textarea %s>%s</textarea>' % (html_params(
         name=field.name, **kwargs), xhtml_escape(to_unicode(
             field._value())))
    def test_complex_debug_page(self):
        response = frontik_no_debug_app.get_page(
            'debug?debug', headers={'Authorization': self.DEBUG_BASIC_AUTH})

        response_content = to_unicode(response.content)

        self.assertEqual(response.status_code, 200)

        # Basic debug messages

        basic_messages = (
            'debug: starting debug page',
            'warning: testing simple inherited debug',
            'error: testing failing urls',
            'info: testing responses',
            'debug mode is ON',
            '<span class="entry__head__expandtext">XSLT profiling results</span>',
        )

        for msg in basic_messages:
            self.assertIn(msg, response_content)

        # Extra output and different types of content

        extra_output = (
            u'&lt;child2&gt;тест&lt;/child2&gt;',
            u'юникод\ndebug',
            u'"тест": "value"',
            u'SomeProtobufObject()',
            u'&lt;response&gt;some xml&lt;/response&gt;',
            u'document.body.write("Привет")',
            u'привет charset',
        )

        for msg in extra_output:
            self.assertIn(msg, response_content)

        # Iframes

        self.assertEqual(response_content.count("doiframe('"), 3)

        # Check that all http requests are present

        self.assertEqual(response_content.count('<div class="timebar">'), 15)

        # Inherited debug

        assert_occurs_twice = (
            'ValueError: Testing an exception',
            '<span class="entry__head__expandtext">Exception traceback</span>',
            '<span class="entry__head__expandtext">testing xml output</span>',
            '<span class="entry__head__expandtext">testing utf-8 text output</span>',
            '<span class="entry__head__expandtext">testing unicode text output</span>',
        )

        for msg in assert_occurs_twice:
            self.assertEqual(response_content.count(msg), 2)

        # Check that everything went right

        assert_not_found = (
            'cannot parse request body',
            'cannot parse response body',
            'cannot append time info',
            'cannot log response info',
            'cannot decode parameter name or value',
            'cannot add traceback lines',
            'error creating log entry with attrs',
            'XSLT debug file error',
        )

        for msg in assert_not_found:
            self.assertNotIn(msg, response_content)
Example #34
0
 def template(self, tmpl, data):
     # Function to generate tornado template.
     return to_unicode(Template(tmpl, autoescape=None).generate(**data))
Example #35
0
 def get_text_response(self):
     return to_unicode(self._response_text)
Example #36
0
 def render_option(cls, value, label, selected):
     options = {'value': value}
     if selected:
         options['selected'] = True
     return '<option %s>%s</option>' % (html_params(**options),
                                        xhtml_escape(to_unicode(label)))
Example #37
0
 def sign_string(self, string_to_sign):
     logger.debug(string_to_sign)
     new_hmac = hmac.new(utf8(self.client.secret_key), digestmod=sha256)
     new_hmac.update(utf8(string_to_sign))
     return to_unicode(b64encode(new_hmac.digest()).rstrip(b'\n'))
Example #38
0
 def _value(self):
     return to_unicode(self.data)
Example #39
0
 def _value(self):
     if self.data:
         return to_unicode(self.data)
     return to_unicode('')
Example #40
0
 def _value(self):
     if self.raw_data:
         return self.raw_data[0]
     elif self.data is not None:
         return to_unicode(self.data)
     return ''
Example #41
0
 def _value(self):
     if self.raw_data:
         return to_unicode(self.raw_data[0])
     return 'y'
Example #42
0
def load_translations(directory: str, encoding: Optional[str] = None) -> None:
    """Loads translations from CSV files in a directory.

    Translations are strings with optional Python-style named placeholders
    (e.g., ``My name is %(name)s``) and their associated translations.

    The directory should have translation files of the form ``LOCALE.csv``,
    e.g. ``es_GT.csv``. The CSV files should have two or three columns: string,
    translation, and an optional plural indicator. Plural indicators should
    be one of "plural" or "singular". A given string can have both singular
    and plural forms. For example ``%(name)s liked this`` may have a
    different verb conjugation depending on whether %(name)s is one
    name or a list of names. There should be two rows in the CSV file for
    that string, one with plural indicator "singular", and one "plural".
    For strings with no verbs that would change on translation, simply
    use "unknown" or the empty string (or don't include the column at all).

    The file is read using the `csv` module in the default "excel" dialect.
    In this format there should not be spaces after the commas.

    If no ``encoding`` parameter is given, the encoding will be
    detected automatically (among UTF-8 and UTF-16) if the file
    contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM
    is present.

    Example translation ``es_LA.csv``::

        "I love you","Te amo"
        "%(name)s liked this","A %(name)s les gustó esto","plural"
        "%(name)s liked this","A %(name)s le gustó esto","singular"

    .. versionchanged:: 4.3
       Added ``encoding`` parameter. Added support for BOM-based encoding
       detection, UTF-16, and UTF-8-with-BOM.
    """
    global _translations
    global _supported_locales
    _translations = {}
    for path in os.listdir(directory):
        if not path.endswith(".csv"):
            continue
        locale, extension = path.split(".")
        if not re.match("[a-z]+(_[A-Z]+)?$", locale):
            gen_log.error(
                "Unrecognized locale %r (path: %s)",
                locale,
                os.path.join(directory, path),
            )
            continue
        full_path = os.path.join(directory, path)
        if encoding is None:
            # Try to autodetect encoding based on the BOM.
            with open(full_path, "rb") as bf:
                data = bf.read(len(codecs.BOM_UTF16_LE))
            if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
                encoding = "utf-16"
            else:
                # utf-8-sig is "utf-8 with optional BOM". It's discouraged
                # in most cases but is common with CSV files because Excel
                # cannot read utf-8 files without a BOM.
                encoding = "utf-8-sig"
        # python 3: csv.reader requires a file open in text mode.
        # Specify an encoding to avoid dependence on $LANG environment variable.
        with open(full_path, encoding=encoding) as f:
            _translations[locale] = {}
            for i, row in enumerate(csv.reader(f)):
                if not row or len(row) < 2:
                    continue
                row = [escape.to_unicode(c).strip() for c in row]
                english, translation = row[:2]
                if len(row) > 2:
                    plural = row[2] or "unknown"
                else:
                    plural = "unknown"
                if plural not in ("plural", "singular", "unknown"):
                    gen_log.error(
                        "Unrecognized plural indicator %r in %s line %d",
                        plural,
                        path,
                        i + 1,
                    )
                    continue
                _translations[locale].setdefault(plural,
                                                 {})[english] = translation
    _supported_locales = frozenset(
        list(_translations.keys()) + [_default_locale])
    gen_log.debug("Supported locales: %s", sorted(_supported_locales))
Example #43
0
 def _value(self):
     if self.raw_data:
         return to_unicode(' '.join(self.raw_data))
     return self.data and self.data.strftime(self.format) or ''
Example #44
0
 def getlist(self, name):
     try:
         return [escape.to_unicode(v) for v in self._wrapped[name]]
     except KeyError:
         return []
Example #45
0
 def assertNotIn(self, observed, expected, *args, **kwargs):
     return super(NBViewerTestCase,
                  self).assertNotIn(to_unicode(observed),
                                    to_unicode(expected), *args, **kwargs)
Example #46
0
 def str_to_unicode(word):
     try:
         return to_unicode(word)
     except Exception, e:
         logger.info(e.message)
         return word.decode("unicode-escape")
Example #47
0
 def test_utf8_in_file(self):
     tmpl = self.loader.load("utf8.html")
     result = tmpl.generate()
     self.assertEqual(to_unicode(result).strip(), u("H\u00e9llo"))
Example #48
0
 def upper(s):
     return utf8(to_unicode(s).upper())
Example #49
0
 def upper(s):
     return to_unicode(s).upper()
Example #50
0
    def __init__(
        self,
        template_string: Union[str, bytes],
        name: str = "<string>",
        loader: Optional["BaseLoader"] = None,
        compress_whitespace: Union[bool, _UnsetMarker] = _UNSET,
        autoescape: Optional[Union[str, _UnsetMarker]] = _UNSET,
        whitespace: Optional[str] = None,
    ) -> None:
        """Construct a Template.

        :arg str template_string: the contents of the template file.
        :arg str name: the filename from which the template was loaded
            (used for error message).
        :arg tornado.template.BaseLoader loader: the `~tornado.template.BaseLoader` responsible
            for this template, used to resolve ``{% include %}`` and ``{% extend %}`` directives.
        :arg bool compress_whitespace: Deprecated since Tornado 4.3.
            Equivalent to ``whitespace="single"`` if true and
            ``whitespace="all"`` if false.
        :arg str autoescape: The name of a function in the template
            namespace, or ``None`` to disable escaping by default.
        :arg str whitespace: A string specifying treatment of whitespace;
            see `filter_whitespace` for options.

        .. versionchanged:: 4.3
           Added ``whitespace`` parameter; deprecated ``compress_whitespace``.
        """
        self.name = escape.native_str(name)

        if compress_whitespace is not _UNSET:
            # Convert deprecated compress_whitespace (bool) to whitespace (str).
            if whitespace is not None:
                raise Exception("cannot set both whitespace and compress_whitespace")
            whitespace = "single" if compress_whitespace else "all"
        if whitespace is None:
            if loader and loader.whitespace:
                whitespace = loader.whitespace
            else:
                # Whitespace defaults by filename.
                if name.endswith(".html") or name.endswith(".js"):
                    whitespace = "single"
                else:
                    whitespace = "all"
        # Validate the whitespace setting.
        assert whitespace is not None
        filter_whitespace(whitespace, "")

        if not isinstance(autoescape, _UnsetMarker):
            self.autoescape = autoescape  # type: Optional[str]
        elif loader:
            self.autoescape = loader.autoescape
        else:
            self.autoescape = _DEFAULT_AUTOESCAPE

        self.namespace = loader.namespace if loader else {}
        reader = _TemplateReader(name, escape.native_str(template_string), whitespace)
        self.file = _File(self, _parse(reader, self))
        self.code = self._generate_python(loader)
        self.loader = loader
        try:
            # Under python2.5, the fake filename used here must match
            # the module name used in __name__ below.
            # The dont_inherit flag prevents template.py's future imports
            # from being applied to the generated code.
            self.compiled = compile(
                escape.to_unicode(self.code),
                "%s.generated.py" % self.name.replace(".", "_"),
                "exec",
                dont_inherit=True,
            )
        except Exception:
            formatted_code = _format_code(self.code).rstrip()
            app_log.error("%s code:\n%s", self.name, formatted_code)
            raise
Example #51
0
 def process_response(self, data):
     status, message = re.match('(.*)\t(.*)\n', to_unicode(data)).groups()
     if status == 'ok':
         return message
     else:
         raise CapError(message)