def CountDownloadBoard(site, version, total_number, pin_number, board_id,
user_id, downloadMethod, atime, user_ip, user_agent):
try:
if user_agent:
# 解析User-Agent
uap = user_agents_parse(user_agent)
browserDevice, browserOs, browserFamily = str(uap).split(' / ')
if uap.is_mobile:
browserType = "mobile"
elif uap.is_pc:
browserType = "pc"
elif uap.is_tablet:
browserType = "tablet"
elif uap.is_bot:
browserType = "bot"
else:
browserType = "unknown"
else:
browserType, browserDevice, browserOs, browserFamily = "", "", "", "", ""
_sb.mysql_write.insert(
"insert into plugin_crawlhuaban_clicklog (site,user_id,board_id,pin_number,total_number,atime,version,download_method,user_ip,user_area,user_agent,browser_type,browser_device,browser_os,browser_family) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
site, user_id, board_id, pin_number,
total_number, atime, version, downloadMethod, user_ip,
getIpArea(user_ip), user_agent, browserType, browserDevice,
browserOs, browserFamily)
except Exception, e:
logger.sys.error(e, exc_info=True)
def Click2MySQL(data):
if isinstance(data, dict):
if "/rqdashboard" in data.get("url") or "/static/" in data.get("url"):
return
if data.get("agent"):
# 解析User-Agent
uap = user_agents_parse(data.get("agent"))
browserDevice, browserOs, browserFamily = str(uap).split(' / ')
if uap.is_mobile:
browserType = "mobile"
elif uap.is_pc:
browserType = "pc"
elif uap.is_tablet:
browserType = "tablet"
elif uap.is_bot:
browserType = "bot"
else:
browserType = "unknown"
sql = "insert into blog_clicklog set url=%s, ip=%s, agent=%s, method=%s, status_code=%s, referer=%s, isp=%s, browserType=%s, browserDevice=%s, browserOs=%s, browserFamily=%s, clickTime=%s"
try:
mid = _sb.mysql_write.insert(sql, data.get("url"),
data.get("ip"), data.get("agent"),
data.get("method"),
data.get("status_code"),
data.get("referer"),
getIpArea(data.get("ip")),
browserType, browserDevice,
browserOs, browserFamily,
get_current_timestamp())
except Exception, e:
logger.plugin.warn(e, exc_info=True)
else:
logger.plugin.debug("Click2MySQL for {}".format(mid))
def DownloadBoard(basedir, board_id, zipfilename, board_pins, total_number, ctime, etime, version, site, user_ip, user_agent):
"""
@param basedir str: 画板上层目录，CrawlHuaban插件所在目录，图片直接保存到此目录的`board_id`下
@param board_id str int: 画板id
@param zipfilename str: 压缩文件名称
@param board_pins list: pin图片列表
@param total_number, ctime, etime int: 分别是画板图片总数、创建时间和过期时间，仅用于写入mysql
@param version, site, user_ip, user_agent str: 分别是用户脚本版本、站点id、用户ip、用户代理，仅用于写入mysql
"""
logger.sys.debug("DownloadBoard dir: {}, board_pins number: {}".format(basedir, len(board_pins)))
#初始写入数据库
try:
if user_agent:
# 解析User-Agent
uap = user_agents_parse(user_agent)
browserDevice, browserOs, browserFamily = str(uap).split(' / ')
if uap.is_mobile:
browserType = "mobile"
elif uap.is_pc:
browserType = "pc"
elif uap.is_tablet:
browserType = "tablet"
elif uap.is_bot:
browserType = "bot"
else:
browserType = "unknown"
else:
browserType, browserDevice, browserOs, browserFamily = "", "", "", "", ""
_sb.mysql_write.insert("insert into plugin_crawlhuaban (site,board_id,filename,pin_number,total_number,ctime,etime,version,user_ip,user_area,user_agent,browser_type,browser_device,browser_os,browser_family) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", site, board_id, zipfilename, len(board_pins), total_number, ctime, etime, version, user_ip, getIpArea(user_ip), user_agent, browserType, browserDevice, browserOs, browserFamily)
except Exception,e:
logger.sys.error(e, exc_info=True)
def parse_userAgent(user_agent):
""" 解析User-Agent """
uap = user_agents_parse(user_agent)
browserDevice, browserOs, browserFamily = str(uap).split(' / ')
if uap.is_mobile:
browserType = "mobile"
elif uap.is_pc:
browserType = "pc"
elif uap.is_tablet:
browserType = "tablet"
elif uap.is_bot:
browserType = "bot"
else:
browserType = "unknown"
return browserType, browserDevice, browserOs, browserFamily
def parse_ua(user_agent):
"""解析用户代理，获取其操作系统、设备、版本"""
uap = user_agents_parse(user_agent)
device, ua_os, family = str(uap).split(' / ')
if uap.is_mobile:
platform = "mobile"
elif uap.is_pc:
platform = "pc"
elif uap.is_tablet:
platform = "tablet"
elif uap.is_bot:
platform = "bot"
else:
platform = "other"
return dict(platform=platform, device=device, os=ua_os, family=family)
def parse_ua(user_agent):
from user_agents import parse as user_agents_parse
uap = user_agents_parse(user_agent)
device, ua_os, family = str(uap).split(' / ')
if uap.is_mobile:
platform = "mobile"
elif uap.is_pc:
platform = "pc"
elif uap.is_tablet:
platform = "tablet"
elif uap.is_bot:
platform = "bot"
else:
platform = "other"
return dict(platform=platform, device=device, os=ua_os, family=family)
def parse_ua(user_agent):
"""解析用户代理，获取其操作系统、设备、版本"""
uap = user_agents_parse(user_agent)
device, ua_os, family = str(uap).split(' / ')
if uap.is_mobile:
platform = "mobile"
elif uap.is_pc:
platform = "pc"
elif uap.is_tablet:
platform = "tablet"
elif uap.is_bot:
platform = "bot"
else:
platform = "other"
if user_agent.startswith("picbed-cli") and user_agent.endswith("amd64"):
_, _, ua_os, device = user_agent.split(" ")
family = "picbed-cli"
return dict(platform=platform, device=device, os=ua_os, family=family)
def human_user_agent(request):
""" Return True if we think we have a human browsing.
The function takes a Django ``request`` as parameter, and will inspect
the ``HTTP_USER_AGENT`` meta header.
.. note:: obviously, if the remote program is faking a real browser user
agent, this function will not do any miracle and will guess it as
a human, whereas it is not.
"""
if user_agents_parse is None:
raise RuntimeError(u'Python module “user-agents” does not '
u'seem to be installed')
user_agent_string = request.META.get('HTTP_USER_AGENT', '')
user_agent = user_agents_parse(user_agent_string)
if user_agent.is_bot:
return False
if user_agent.is_mobile or user_agent.is_tablet:
return True
if user_agent.is_pc:
if user_agent.is_touch_capable:
return True
if user_agent.browser.family in (
'Firefox',
'Chrome',
):
return True
#
# TODO: finish this…
#
return False