def _escape(self, value, escape=False):
import types
if value is None:
value = ''
elif isinstance(value, types.GeneratorType):
value = self._join_output(value)
value = safeunicode(value)
if escape and self.filter:
value = self.filter(value)
return safeunicode(value)
def render(self):
out = ""
out += self.rendernote(self.note)
# out += '<table>\n'
for i in self.inputs:
html = utils.safeunicode(i.pre) + i.render() + self.rendernote(i.note) + utils.safeunicode(i.post)
if i.is_hidden():
out += " %s\n" % (html)
else:
out += ' <label for="%s">%s</label>%s\n' % (i.id, net.websafe(i.description), html)
# out += "</table>"
return out
def render(self):
out = ''
out += self.rendernote(self.note)
out += '<table>\n'
for i in self.inputs:
html = utils.safeunicode(i.pre) + i.render() + self.rendernote(i.note) + utils.safeunicode(i.post)
if i.is_hidden():
out += ' <tr style="display: none;"><th></th><td>%s</td></tr>\n' % (html)
else:
out += ' <tr><th><label for="%s">%s</label></th><td>%s</td></tr>\n' % (i.id, net.websafe(i.description), html)
out += "</table>"
return out
def render(self):
out = ''
out += self.rendernote(self.note)
out += '<table>\n'
for i in self.inputs:
html = utils.safeunicode(i.pre) + i.render() + self.rendernote(i.note) + utils.safeunicode(i.post)
if i.is_hidden():
out += ' <tr style="display: none;"><th></th><td>%s</td></tr>\n' % (html)
else:
# sanitation removed by KBP because I'm lazy and this isn't a real webapp anyways
#out += ' <tr><th><label for="%s">%s</label></th><td>%s</td></tr>\n' % (i.id, net.websafe(i.description), html)
out += ' <tr><th><label for="%s">%s</label></th><td>%s</td></tr>\n' % (i.id, i.description, html)
out += "</table>"
return out
def debug(message, **kwargs):
if not isinstance(message, unicode):
message = safeunicode(message)
if EVENT_DEBUG in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_DEBUG, message, **kwargs)
if EVENT_TRACE in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_TRACE, 'debug', message, **kwargs)
def error(message, **kwargs):
if not isinstance(message, unicode):
message = safeunicode(message)
if EVENT_ERROR in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_ERROR, message, **kwargs)
if EVENT_TRACE in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_TRACE, 'error', message, **kwargs)
def load(self, env):
"""Initializes ctx using env."""
ctx = web.ctx
ctx.status = '200 OK'
ctx.headers = []
ctx.output = ''
ctx.environ = ctx.env = env
ctx.host = env.get('HTTP_HOST')
ctx.protocol = env.get('HTTPS') and 'https' or 'http'
ctx.homedomain = ctx.protocol + '://' + env.get('HTTP_HOST', '[unknown]')
ctx.homepath = os.environ.get('REAL_SCRIPT_NAME', env.get('SCRIPT_NAME', ''))
ctx.home = ctx.homedomain + ctx.homepath
#@@ home is changed when the request is handled to a sub-application.
#@@ but the real home is required for doing absolute redirects.
ctx.realhome = ctx.home
ctx.ip = env.get('REMOTE_ADDR')
ctx.method = env.get('REQUEST_METHOD')
ctx.path = env.get('PATH_INFO')
# http://trac.lighttpd.net/trac/ticket/406 requires:
if env.get('SERVER_SOFTWARE', '').startswith('lighttpd/'):
ctx.path = lstrips(env.get('REQUEST_URI').split('?')[0], ctx.homepath)
if env.get('QUERY_STRING'):
ctx.query = '?' + env.get('QUERY_STRING', '')
else:
ctx.query = ''
ctx.fullpath = ctx.path + ctx.query
for k, v in ctx.iteritems():
if isinstance(v, str):
ctx[k] = safeunicode(v)
# status must always be str
ctx.status = '200 OK'
def info(message, trace='info', **kwargs):
if not isinstance(message, unicode):
message = safeunicode(message)
if EVENT_INFO in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_INFO, message, **kwargs)
if EVENT_TRACE in dispatch.dispatch.callbacks:
dispatch.pub(EVENT_TRACE, trace, message, **kwargs)
def _escape(self, value, escape=False):
if value is None:
value = ''
value = safeunicode(value)
if escape and self.filter:
value = self.filter(value)
return value
def _escape(self, value, escape=False):
if value is None:
value = ''
value = safeunicode(value)
if escape and self.filter:
value = self.filter(value)
return value
def specialFilter(self):
if len(self.filters) > 0:
for filter in self.filters:
rule = filter;
rule = rule.replace('(*)', '(.+)?')
if isinstance(self.content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
self.content = re.compile(rule, re.I).sub("", self.content);
def specialFilter(self):
if len(self.filters) > 0:
for filter in self.filters:
rule = filter
rule = rule.replace('(*)', '(.+)?')
if isinstance(self.content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
self.content = re.compile(rule, re.I).sub("", self.content)
def render(self):
out = ''
if self.js:
out = self.js.render()
out += self.rendernote(self.note)
out += '<table>\n<tr>'
for i in self.inputs:
html = utils.safeunicode(i.pre) + i.render() + self.rendernote(i.note) + utils.safeunicode(i.post)
if i.is_hidden():
out += ' <th></th><td>%s</td>\n' % (html)
elif i.has_no_name():
out += ' <td>%s</td>\n' % (html)
else:
out += ' <th><label for="%s">%s</label></th><td>%s</td>\n' % (i.id, net.websafe(i.description), html)
out += "</tr></table>"
return out
def _escape(self, value, escape=False):
import types
if value is None:
value = ''
elif isinstance(value, types.GeneratorType):
value = self._join_output(value)
value = safeunicode(value)
if escape and self.filter:
value = self.filter(value)
return value
def make_sign(api_secret, params=[]):
"""
>>> make_sign("123456",[1,'2',u'中文'])
'33C9065427EECA3490C5642C99165145'
"""
_params = [utils.safeunicode(p) for p in params if p is not None]
_params.sort()
_params.insert(0, api_secret)
strs = ''.join(_params)
mds = md5(strs.encode('utf-8')).hexdigest()
return mds.upper()
def load(self, env):
"""Initializes ctx using env."""
ctx = web.ctx
ctx.clear()
ctx.status = '200 OK'
ctx.headers = []
ctx.output = ''
ctx.environ = ctx.env = env
ctx.host = env.get('HTTP_HOST')
if env.get('wsgi.url_scheme') in ['http', 'https']:
ctx.protocol = env['wsgi.url_scheme']
elif env.get('HTTPS', '').lower() in ['on', 'true', '1']:
ctx.protocol = 'https'
else:
ctx.protocol = 'http'
ctx.homedomain = ctx.protocol + '://' + env.get(
'HTTP_HOST', '[unknown]')
ctx.homepath = os.environ.get('REAL_SCRIPT_NAME',
env.get('SCRIPT_NAME', ''))
ctx.home = ctx.homedomain + ctx.homepath
#@@ home is changed when the request is handled to a sub-application.
#@@ but the real home is required for doing absolute redirects.
ctx.realhome = ctx.home
ctx.ip = env.get('REMOTE_ADDR')
ctx.method = env.get('REQUEST_METHOD')
ctx.path = env.get('PATH_INFO')
# http://trac.lighttpd.net/trac/ticket/406 requires:
if env.get('SERVER_SOFTWARE', '').startswith('lighttpd/'):
ctx.path = lstrips(
env.get('REQUEST_URI').split('?')[0], ctx.homepath)
# Apache and CherryPy webservers unquote the url but lighttpd doesn't.
# unquote explicitly for lighttpd to make ctx.path uniform across all servers.
ctx.path = urllib.unquote(ctx.path)
if env.get('QUERY_STRING'):
ctx.query = '?' + env.get('QUERY_STRING', '')
else:
ctx.query = ''
ctx.fullpath = ctx.path + ctx.query
for k, v in ctx.iteritems():
if isinstance(v, str):
ctx[k] = safeunicode(v)
# status must always be str
ctx.status = '200 OK'
ctx.app_stack = []
def emit(self, eventDict):
text = txlog.textFromEventDict(eventDict)
if text is None:
return
else:
if not isinstance(text, (unicode, str, dict, list)):
text = text
else:
text = safeunicode(text)
if eventDict['isError'] and 'failure' in eventDict:
self.exception(text)
else:
self.info(text)
return
def load(self, env):
"""Initializes ctx using env."""
ctx = web.ctx
ctx.clear()
ctx.status = "200 OK"
ctx.headers = []
ctx.output = ""
ctx.environ = ctx.env = env
ctx.host = env.get("HTTP_HOST")
if env.get("wsgi.url_scheme") in ["http", "https"]:
ctx.protocol = env["wsgi.url_scheme"]
elif env.get("HTTPS", "").lower() in ["on", "true", "1"]:
ctx.protocol = "https"
else:
ctx.protocol = "http"
ctx.homedomain = ctx.protocol + "://" + env.get("HTTP_HOST", "[unknown]")
ctx.homepath = os.environ.get("REAL_SCRIPT_NAME", env.get("SCRIPT_NAME", ""))
ctx.home = ctx.homedomain + ctx.homepath
# @@ home is changed when the request is handled to a sub-application.
# @@ but the real home is required for doing absolute redirects.
ctx.realhome = ctx.home
ctx.ip = env.get("REMOTE_ADDR")
ctx.method = env.get("REQUEST_METHOD")
ctx.path = env.get("PATH_INFO")
# http://trac.lighttpd.net/trac/ticket/406 requires:
if env.get("SERVER_SOFTWARE", "").startswith("lighttpd/"):
ctx.path = lstrips(env.get("REQUEST_URI").split("?")[0], ctx.homepath)
# Apache and CherryPy webservers unquote the url but lighttpd doesn't.
# unquote explicitly for lighttpd to make ctx.path uniform across all servers.
ctx.path = urllib.unquote(ctx.path)
if env.get("QUERY_STRING"):
ctx.query = "?" + env.get("QUERY_STRING", "")
else:
ctx.query = ""
ctx.fullpath = ctx.path + ctx.query
for k, v in ctx.iteritems():
if isinstance(v, str):
ctx[k] = safeunicode(v)
# status must always be str
ctx.status = "200 OK"
ctx.app_stack = []
def load(self, env):
"""Initializes ctx using env."""
ctx = web.ctx
ctx.clear()
ctx.status = '200 OK'
ctx.headers = []
ctx.output = ''
ctx.environ = ctx.env = env
ctx.host = env.get('HTTP_HOST')
if env.get('wsgi.url_scheme') in ['http', 'https']:
ctx.protocol = env['wsgi.url_scheme']
elif env.get('HTTPS', '').lower() in ['on', 'true', '1']:
ctx.protocol = 'https'
else:
ctx.protocol = 'http'
ctx.homedomain = ctx.protocol + '://' + env.get('HTTP_HOST', '[unknown]')
ctx.homepath = os.environ.get('REAL_SCRIPT_NAME', env.get('SCRIPT_NAME', ''))
ctx.home = ctx.homedomain + ctx.homepath
#@@ home is changed when the request is handled to a sub-application.
#@@ but the real home is required for doing absolute redirects.
ctx.realhome = ctx.home
ctx.ip = env.get('REMOTE_ADDR')
ctx.method = env.get('REQUEST_METHOD')
ctx.path = env.get('PATH_INFO')
# http://trac.lighttpd.net/trac/ticket/406 requires:
if env.get('SERVER_SOFTWARE', '').startswith('lighttpd/'):
ctx.path = lstrips(env.get('REQUEST_URI').split('?')[0], ctx.homepath)
# Apache and CherryPy webservers unquote the url but lighttpd doesn't.
# unquote explicitly for lighttpd to make ctx.path uniform across all servers.
ctx.path = urllib.unquote(ctx.path)
if env.get('QUERY_STRING'):
ctx.query = '?' + env.get('QUERY_STRING', '')
else:
ctx.query = ''
ctx.fullpath = ctx.path + ctx.query
for k, v in ctx.iteritems():
if isinstance(v, str):
ctx[k] = safeunicode(v)
# status must always be str
ctx.status = '200 OK'
ctx.app_stack = []
def __call__(self, item):
output_fd = self._get_fd(item)
try:
if isinstance(item, (types.ListType, types.TupleType)):
print(self._delimiter.join([str(i) for i in item]), file=output_fd)
elif isinstance(item, types.DictionaryType):
print(json.dumps(item), file=output_fd)
elif isinstance(item, types.StringTypes):
# 注意: print会自带一个回车, 默认读入的文件数据是有\n的
output_fd.write(utils.safeunicode(item))
else:
raise UnsupportedTypeException("Data type is not supported. [type={t} item={i}]"
.format(t=type(item), i=item))
except (KeyboardInterrupt, UnsupportedTypeException) as e:
logger.warn("Unsupported action or user cancelled. [item={item} exception={exc}]"
.format(item=item, exc=traceback.format_exc()))
raise e
except:
logger.warn("Unknown exception. [item={item} exception={exc}]"
.format(item=item, exc=traceback.format_exc()))
def emit(self, indent, begin_indent=''):
return repr(safeunicode(self.value))
def emit(self, indent, begin_indent=''):
return repr(safeunicode(self.value))
def __unicode__(self):
return safeunicode(self._str())
def getElementData(obj, rule, images=None, fetch_all=0):
"""
根据rule对obj的进行解析
obj可以是pq后的对象， 也可以是html页面
images将会把解析过程的image连接插入此表中
规则可以有两种模式：
1. DOM selector
1.1 选择器类似于jquery 比如你要某个a的url
>> a.attr("href")
1.2 需要一个标签内的文本内容
>> div[id="content"].text()
1.3 需要获得某个子元素中的内容
>> li.eq(1).text() #li元素组中的第2个文本内容
2. 正则模式
正则模式需要的内容使用[arg]标签，其余可以使用(*)填充
"""
if not isinstance(obj, pq):
obj = pq(obj)
old_rule = rule
rule = rule.split(".")
#避免有url链接
if len(rule) > 1 and old_rule.find("[arg]") == -1:
#第一个永远是dom选择
selectRule = rule.pop(0)
#移除 ( )
selectRule = selectRule.replace("(", "")
selectRule = selectRule.replace(")", "")
selecteddom = obj.find(selectRule)
for attr in rule:
m = attrParrent.match(attr)
if m:
action, v = m.groups()
if v:
v = v.encode("utf-8")
#去除引号
v = v.strip("\'").strip('\"')
if action == "attr" and hasattr(selecteddom, "attr") and v:
if fetch_all == 1:
values = []
dom_count = len(selecteddom)
for i in range(dom_count):
vv = selecteddom.eq(i).attr(v)
if vv:
values.append(vv)
if is_image(vv):
images.append(vv)
return values
else:
value = selecteddom.attr(v)
if selecteddom and selecteddom[
0].tag == "img" and v == "src" and images is not None:
images.append(value)
return value
elif action == "eq" and hasattr(selecteddom, "eq"):
_rules = attr.split(" ")
if len(rule) > 1:
selecteddom = selecteddom.eq(int(v))
if len(_rules) > 1:
'''
假设eq后面还有子元素
eq(1) a
'''
_rules.pop(0)
_dom = " ".join(_rules)
selecteddom = selecteddom.find(_dom)
else:
return selecteddom.eq(int(v))
elif action == "text" and hasattr(selecteddom, "text"):
return safeunicode(selecteddom.text()).strip()
elif action == "html" and hasattr(selecteddom, "html"):
return safeunicode(selecteddom.html()).strip()
elif len(rule) == 1:
rule = rule.pop()
#正则模式
if rule.find('[arg]'):
content = obj.html()
content_text = obj.text()
rule = rule.replace('[arg]', '(.+)?')
rule = rule.replace('(*)', '.+?')
if isinstance(content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
parrent = re.compile(rule, re.MULTILINE | re.UNICODE)
try:
result = parrent.search(content)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
else:
result = parrent.search(content_text)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
except:
return None
return None
def __unicode__(self):
return safeunicode(self.get("__body__", ""))
def __unicode__(self):
return safeunicode(self._str())
def __unicode__(self):
return safeunicode(self.get('__body__', ''))
def emit(self, indent):
return repr(safeunicode(self.value))
def _join(self, *items):
return u"".join([safeunicode(item) for item in items])
def emit(self, indent):
return repr(safeunicode(self.value))
def getElementData(obj, rule, images=None, fetch_all=0):
"""
根据rule对obj的进行解析
obj可以是pq后的对象， 也可以是html页面
images将会把解析过程的image连接插入此表中
规则可以有两种模式：
1. DOM selector
1.1 选择器类似于jquery 比如你要某个a的url
>> a.attr("href")
1.2 需要一个标签内的文本内容
>> div[id="content"].text()
1.3 需要获得某个子元素中的内容
>> li.eq(1).text() #li元素组中的第2个文本内容
2. 正则模式
正则模式需要的内容使用[arg]标签，其余可以使用(*)填充
"""
if not isinstance(obj, pq):
obj = pq(obj);
old_rule = rule
rule = rule.split(".")
#避免有url链接
if len(rule) > 1 and old_rule.find("[arg]") == -1:
#第一个永远是dom选择
selectRule = rule.pop(0)
#移除 ( )
selectRule = selectRule.replace("(", "");
selectRule = selectRule.replace(")", "");
selecteddom = obj.find(selectRule);
for attr in rule:
m = attrParrent.match(attr)
if m:
action, v = m.groups()
if v:
v = v.encode("utf-8")
#去除引号
v = v.strip("\'").strip('\"');
if action == "attr" and hasattr(selecteddom, "attr") and v:
if fetch_all == 1:
values = []
dom_count = len(selecteddom)
for i in range(dom_count):
vv = selecteddom.eq(i).attr(v)
if vv:
values.append(vv)
if is_image(vv):
images.append(vv)
return values
else:
value = selecteddom.attr(v)
if selecteddom and selecteddom[0].tag == "img" and v == "src" and images is not None:
images.append(value)
return value
elif action == "eq" and hasattr(selecteddom, "eq"):
_rules = attr.split(" ")
if len(rule) > 1:
selecteddom = selecteddom.eq(int(v))
if len(_rules) > 1:
'''
假设eq后面还有子元素
eq(1) a
'''
_rules.pop(0)
_dom = " ".join(_rules)
selecteddom = selecteddom.find(_dom)
else:
return selecteddom.eq(int(v))
elif action == "text" and hasattr(selecteddom, "text"):
return safeunicode(selecteddom.text()).strip()
elif action == "html" and hasattr(selecteddom, "html"):
return safeunicode(selecteddom.html()).strip()
elif len(rule) == 1:
rule = rule.pop()
#正则模式
if rule.find('[arg]'):
content = obj.html()
content_text = obj.text()
rule = rule.replace('[arg]', '(.+)?')
rule = rule.replace('(*)', '.+?')
if isinstance(content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
parrent = re.compile(rule, re.MULTILINE | re.UNICODE)
try:
result = parrent.search(content)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
else:
result = parrent.search(content_text)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
except:
return None
return None
def _join(self, *items):
return u"".join([safeunicode(item) for item in items])
def __unicode__(self):
return safeunicode(self.get('__body__', ''))
