def setcookie(name, value, expires='', domain=None,
secure=False, httponly=False, path=None):
"""Sets a cookie."""
morsel = Cookie.Morsel()
name, value = safestr(name), safestr(value)
morsel.set(name, value, urllib.quote(value))
if expires < 0:
expires = -1000000000
morsel['expires'] = expires
#"""
if path:
morsel['path'] = path
else:
if ctx.homepath == '':
morsel['path'] = '/'
else:
morsel['path'] = ctx.homepath
#"""
#morsel['path'] = path or ctx.homepath+'/'
if domain:
morsel['domain'] = domain
if secure:
morsel['secure'] = secure
value = morsel.OutputString()
if httponly:
value += '; httponly'
header('Set-Cookie', value)
def header(hdr, value, unique=False):
hdr, value = safestr(hdr), safestr(value)
if '\n' in hdr or '\r' in hdr or '\n' in value or '\r' in value:
raise ValueError, "invalid characters in header"
if unique is True:
for h, v in ctx.headers:
if h.lower() == hdr.lower(): return
ctx.headers.append((hdr, value))
def query(self, paramstyle=None):
s = []
for x in self.items:
if isinstance(x, SQLParam):
x = x.get_marker(paramstyle)
s.append(safestr(x))
else:
x = safestr(x)
if paramstyle in ['format', 'pyformat']:
if '%' in x and '%%' not in x:
x = x.replace('%', '%%')
s.append(x)
return "".join(s)
def generate_code(text, filename):
# parse the text
rootnode = Parser(text, filename).parse()
# generate python code from the parse tree
code = rootnode.emit(indent="").strip()
return safestr(code)
def header(hdr, value, unique=False):
"""
Adds the header `hdr: value` with the response.
If `unique` is True and a header with that name already exists,
it doesn't add a new one.
"""
hdr, value = safestr(hdr), safestr(value)
# protection against HTTP response splitting attack
if '\n' in hdr or '\r' in hdr or '\n' in value or '\r' in value:
raise ValueError, 'invalid characters in header'
if unique is True:
for h, v in ctx.headers:
if h.lower() == hdr.lower(): return
ctx.headers.append((hdr, value))
def generate_code(text, filename, parser=None):
# parse the text
parser = parser or Parser()
rootnode = parser.parse(text, filename)
# generate python code from the parse tree
code = rootnode.emit(indent="").strip()
return safestr(code)
def _render_option(self, arg, indent=' '):
if isinstance(arg, (tuple, list)):
value, desc= arg
else:
value, desc = arg, arg
value = utils.safestr(value)
if isinstance(self.value, (tuple, list)):
s_value = [utils.safestr(x) for x in self.value]
else:
s_value = utils.safestr(self.value)
if s_value == value or (isinstance(s_value, list) and value in s_value):
select_p = ' selected="selected"'
else:
select_p = ''
return indent + '<option%s value="%s">%s</option>\n' % (select_p, net.websafe(value), net.websafe(desc))
def specialFilter(self):
if len(self.filters) > 0:
for filter in self.filters:
rule = filter;
rule = rule.replace('(*)', '(.+)?')
if isinstance(self.content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
self.content = re.compile(rule, re.I).sub("", self.content);
def log(self, status, environ):
outfile = environ.get('wsgi.errors', web.debug)
req = environ.get('PATH_INFO', '_')
protocol = environ.get('ACTUAL_SERVER_PROTOCOL', '-')
method = environ.get('REQUEST_METHOD', '-')
host = "%s:%s"% (environ.get('REMOTE_ADDR', '-'),
environ.get('REMOTE_PORT', '-'))
time = self.log_date_time_string()
msg = self.format% (host, time, protocol, method, req, status)
print >> outfile, utils.safestr(msg)
def _generate_session_id(self):
while True:
rand = os.urandom(16)
now = time.time()
secret_key = self._config.secret_key
session_id = sha1("%s%s%s%s" %(rand, now, utils.safestr(web.ctx.ip), secret_key))
session_id = session_id.hexdigest()
if session_id not in self.store:
break
return session_id
def log(self, status, environ):
outfile = environ.get("wsgi.errors", web.debug)
req = environ.get("PATH_INFO", "_")
protocol = environ.get("ACTUAL_SERVER_PROTOCOL", "-")
method = environ.get("REQUEST_METHOD", "-")
host = "%s:%s" % (environ.get("REMOTE_ADDR", "-"), environ.get("REMOTE_PORT", "-"))
time = self.log_date_time_string()
msg = self.format % (host, time, protocol, method, req, status)
print >> outfile, utils.safestr(msg)
def log(self, status, environ):
outfile = environ.get('wsgi.errors', web.debug)
req = environ.get('PATH_INFO', '_')
protocol = environ.get('ACTUAL_SERVER_PROTOCOL', '-')
method = environ.get('REQUEST_METHOD', '-')
host = "%s:%s" % (environ.get('REMOTE_ADDR','-'),
environ.get('REMOTE_PORT','-'))
#@@ It is really bad to extend from
#@@ BaseHTTPRequestHandler just for this method
time = self.log_date_time_string()
msg = self.format % (host, time, protocol, method, req, status)
print >> outfile, utils.safestr(msg)
def query(self, paramstyle=None):
"""
Returns the query part of the sql query.
>>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')])
>>> q.query()
'SELECT * FROM test WHERE name=%s'
>>> q.query(paramstyle='qmark')
'SELECT * FROM test WHERE name=?'
"""
s = []
for x in self.items:
if isinstance(x, SQLParam):
x = x.get_marker(paramstyle)
s.append(safestr(x))
else:
x = safestr(x)
# automatically escape % characters in the query
# For backward compatability, ignore escaping when the query looks already escaped
if paramstyle in ['format', 'pyformat']:
if '%' in x and '%%' not in x:
x = x.replace('%', '%%')
s.append(x)
return "".join(s)
def query(self, paramstyle=None):
"""
Returns the query part of the sql query.
>>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')])
>>> q.query()
'SELECT * FROM test WHERE name=%s'
>>> q.query(paramstyle='qmark')
'SELECT * FROM test WHERE name=?'
"""
s = ''
for x in self.items:
if isinstance(x, SQLParam):
x = x.get_marker(paramstyle)
s += safestr(x)
return s
def fetchListPages(self, listtype="html"):
print "Start to fetch and parse List"
urls = self.listRule.getListUrls()
for url in urls:
print "Fetching list page: ", url, "charset:", safestr(self.seed["charset"]), "timeout:", safestr(self.seed["timeout"])
f = Fetch(url, charset = self.seed["charset"], timeout = self.seed["timeout"])
if f.isReady():
doc = f.read()
if listtype == "html":
self.parseListPage(f, doc, url)
elif listtype == "json":
self.parseJsonPage(f, doc, url)
print "List has finished parsing. It has %s docs." % ansicolor.red(self.__len__())
def __call__(self, env, start_response):
for path,what in self.handlers:
match = path.match(env['PATH_INFO'])
if match:
handle = what(env, self.render)
args = [x for x in match.groups()]
try:
result = getattr(handle, env['REQUEST_METHOD'])(*args)
except:
print sys.stderr, traceback.format_exc()
if not self.debug:
result = "Internal Server Error"
else:
result = traceback.format_exc()
handle.response.status = "500 Internal Server Error"
#for x in handle.response.headers:
# if x[0] == "Content-Type":
# handle.response.headers.remove(x)
# handle.response.headers.append(('Content-Type', "text/plain; charset=UTF-8"))
# break
#if "Content-Length" not in [k for k,_ in handle.response.headers]:
# handle.response.headers.append(('Content-Length', len(result)))
#start_response(handle.response.status, [(k,v) for (k,v) in handle.response.headers.items()])
start_response(handle.response.status, handle.response.headers)
if hasattr(result, "next"):
for x in result:
yield safestr(x)
else:
yield safestr(result)
handle.cleanup()
env.clear()
handle = None
return
start_response('404 Not Found', [('Content-Type', 'text/html')])
yield ['<h1>Not Found</h1>']
def setcookie(name, value, expires="", domain=None, secure=False, httponly=False):
"""Sets a cookie."""
if expires < 0:
expires = -1000000000
kargs = {"expires": expires, "path": "/"}
if domain:
kargs["domain"] = domain
if secure:
kargs["secure"] = secure
# @@ should we limit cookies to a different path?
cookie = Cookie.SimpleCookie()
cookie[name] = urllib.quote(safestr(value))
for key, val in kargs.iteritems():
cookie[name][key] = val
value = cookie.items()[0][1].OutputString()
if httponly:
value += "; httponly"
header("Set-Cookie", value)
def emailerrors_internal():
error = olderror()
tb = sys.exc_info()
error_name = tb[0]
error_value = tb[1]
tb_txt = ''.join(traceback.format_exception(*tb))
path = web.ctx.path
request = web.ctx.method + ' ' + web.ctx.home + web.ctx.fullpath
message = "\n%s\n\n%s\n\n" % (request, tb_txt)
sendmail(
"your buggy site <%s>" % from_address,
"the bugfixer <%s>" % to_address,
"bug: %(error_name)s: %(error_value)s (%(path)s)" % locals(),
message,
attachments=[
dict(filename="bug.html", content=safestr(djangoerror()))
],
)
return error
def get_inner_dl(self):
return "{}/{}.apk".format(safestr(self.inner_host),
safestr(self.alias))
def get_default_id(self):
value = utils.safestr(self.value or "")
return self.name + '_' + value.replace(' ', '_')
def getElementData(obj, rule, images=None, fetch_all=0):
"""
根据rule对obj的进行解析
obj可以是pq后的对象， 也可以是html页面
images将会把解析过程的image连接插入此表中
规则可以有两种模式：
1. DOM selector
1.1 选择器类似于jquery 比如你要某个a的url
>> a.attr("href")
1.2 需要一个标签内的文本内容
>> div[id="content"].text()
1.3 需要获得某个子元素中的内容
>> li.eq(1).text() #li元素组中的第2个文本内容
2. 正则模式
正则模式需要的内容使用[arg]标签，其余可以使用(*)填充
"""
if not isinstance(obj, pq):
obj = pq(obj);
old_rule = rule
rule = rule.split(".")
#避免有url链接
if len(rule) > 1 and old_rule.find("[arg]") == -1:
#第一个永远是dom选择
selectRule = rule.pop(0)
#移除 ( )
selectRule = selectRule.replace("(", "");
selectRule = selectRule.replace(")", "");
selecteddom = obj.find(selectRule);
for attr in rule:
m = attrParrent.match(attr)
if m:
action, v = m.groups()
if v:
v = v.encode("utf-8")
#去除引号
v = v.strip("\'").strip('\"');
if action == "attr" and hasattr(selecteddom, "attr") and v:
if fetch_all == 1:
values = []
dom_count = len(selecteddom)
for i in range(dom_count):
vv = selecteddom.eq(i).attr(v)
if vv:
values.append(vv)
if is_image(vv):
images.append(vv)
return values
else:
value = selecteddom.attr(v)
if selecteddom and selecteddom[0].tag == "img" and v == "src" and images is not None:
images.append(value)
return value
elif action == "eq" and hasattr(selecteddom, "eq"):
_rules = attr.split(" ")
if len(rule) > 1:
selecteddom = selecteddom.eq(int(v))
if len(_rules) > 1:
'''
假设eq后面还有子元素
eq(1) a
'''
_rules.pop(0)
_dom = " ".join(_rules)
selecteddom = selecteddom.find(_dom)
else:
return selecteddom.eq(int(v))
elif action == "text" and hasattr(selecteddom, "text"):
return safeunicode(selecteddom.text()).strip()
elif action == "html" and hasattr(selecteddom, "html"):
return safeunicode(selecteddom.html()).strip()
elif len(rule) == 1:
rule = rule.pop()
#正则模式
if rule.find('[arg]'):
content = obj.html()
content_text = obj.text()
rule = rule.replace('[arg]', '(.+)?')
rule = rule.replace('(*)', '.+?')
if isinstance(content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
parrent = re.compile(rule, re.MULTILINE | re.UNICODE)
try:
result = parrent.search(content)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
else:
result = parrent.search(content_text)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
except:
return None
return None
def __str__(self):
return safestr(self._str())
def __str__(self):
return safestr(self.get('__body__', ''))
def convert(value, doseq=False):
if doseq and isinstance(value, list):
return [convert(v) for v in value]
else:
return utils.safestr(value)
def get_default_id(self):
value = utils.safestr(self.value or "")
return self.name + "_" + value.replace(" ", "_")
def getElementData(obj, rule, images=None, fetch_all=0):
"""
根据rule对obj的进行解析
obj可以是pq后的对象， 也可以是html页面
images将会把解析过程的image连接插入此表中
规则可以有两种模式：
1. DOM selector
1.1 选择器类似于jquery 比如你要某个a的url
>> a.attr("href")
1.2 需要一个标签内的文本内容
>> div[id="content"].text()
1.3 需要获得某个子元素中的内容
>> li.eq(1).text() #li元素组中的第2个文本内容
2. 正则模式
正则模式需要的内容使用[arg]标签，其余可以使用(*)填充
"""
if not isinstance(obj, pq):
obj = pq(obj)
old_rule = rule
rule = rule.split(".")
#避免有url链接
if len(rule) > 1 and old_rule.find("[arg]") == -1:
#第一个永远是dom选择
selectRule = rule.pop(0)
#移除 ( )
selectRule = selectRule.replace("(", "")
selectRule = selectRule.replace(")", "")
selecteddom = obj.find(selectRule)
for attr in rule:
m = attrParrent.match(attr)
if m:
action, v = m.groups()
if v:
v = v.encode("utf-8")
#去除引号
v = v.strip("\'").strip('\"')
if action == "attr" and hasattr(selecteddom, "attr") and v:
if fetch_all == 1:
values = []
dom_count = len(selecteddom)
for i in range(dom_count):
vv = selecteddom.eq(i).attr(v)
if vv:
values.append(vv)
if is_image(vv):
images.append(vv)
return values
else:
value = selecteddom.attr(v)
if selecteddom and selecteddom[
0].tag == "img" and v == "src" and images is not None:
images.append(value)
return value
elif action == "eq" and hasattr(selecteddom, "eq"):
_rules = attr.split(" ")
if len(rule) > 1:
selecteddom = selecteddom.eq(int(v))
if len(_rules) > 1:
'''
假设eq后面还有子元素
eq(1) a
'''
_rules.pop(0)
_dom = " ".join(_rules)
selecteddom = selecteddom.find(_dom)
else:
return selecteddom.eq(int(v))
elif action == "text" and hasattr(selecteddom, "text"):
return safeunicode(selecteddom.text()).strip()
elif action == "html" and hasattr(selecteddom, "html"):
return safeunicode(selecteddom.html()).strip()
elif len(rule) == 1:
rule = rule.pop()
#正则模式
if rule.find('[arg]'):
content = obj.html()
content_text = obj.text()
rule = rule.replace('[arg]', '(.+)?')
rule = rule.replace('(*)', '.+?')
if isinstance(content, unicode):
rule = safeunicode(rule)
else:
rule = safestr(rule)
parrent = re.compile(rule, re.MULTILINE | re.UNICODE)
try:
result = parrent.search(content)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
else:
result = parrent.search(content_text)
if result is not None:
result = safeunicode(result.group(1)).strip()
return result
except:
return None
return None
def convert(value, doseq=False):
if doseq and isinstance(value, list):
return [convert(v) for v in value]
else:
return utils.safestr(value)
def __str__(self):
return safestr(self.get("__body__", ""))
def __str__(self):
return safestr(self._str())
def __str__(self):
return safestr(self.get('__body__', ''))
def get_default_id(self):
value = utils.safestr(self.value or "")
return self.name + '_' + value.replace(' ', '_')