def jd_crawler():
pid = request.values.get('pid', 0)
# 验证码忽略大小写
code = str(request.values.get('captcha', 0)).lower()
# 判断验证码是否正确
if code != session.get('captcha'):
result = {'result': 'wrong_code'}
return json.dumps(result)
# 查询数据库中是否已存在该商品
product = get_product_by_pid(pid)
if product is None:
# 若无，将获取的pid与网址前后缀连接，插入到任务队列
redis_client.lpush('jd:items_urls', PREFIX + pid + POSTFIX)
# 每两秒查询一次，爬取任务是否完成
while int(redis_client.get(pid)) > 0:
time.sleep(2)
elif len(product.comments) < WORK_LOAD:
# 说明当前爬取任务正在进行中
# 每两秒查询一次，爬取任务是否完成
while int(redis_client.get(pid)) > 0:
time.sleep(2)
result = {'result': 'ok', 'pid': pid}
return json.dumps(result)
def jd_crawler():
pid = request.values.get('pid', 0)
# 查询数据库中是否已存在该商品
product = get_product_by_pid(pid)
if product is None:
# 若无，将获取的pid与网址前后缀连接，插入到任务队列
redis_client.lpush('jd:items_urls', PREFIX + pid + POSTFIX)
# 每两秒查询一次，爬取任务是否完成
while int(redis_client.get(pid)) > 0:
time.sleep(2)
else:
# 说明当前爬取任务正在进行中
# 每两秒查询一次，爬取任务是否完成
while int(redis_client.get(pid)) > 0:
time.sleep(2)
result = {'result': 'ok', 'pid': pid}
return json.dumps(result)
def jd_charts(chart, pid):
product = get_product_by_pid(pid)
if product is None:
return '404!!!'
jd_page = JDPage(product)
chart_type = 1
if chart == 'bar':
jd_page.generate_bar_charts()
elif chart == 'pie':
jd_page.generate_pie_charts()
chart_type = 2
elif chart == 'wordcloud':
jd_page.generate_word_cloud_charts()
chart_type = 3
else:
return '404!'
return render_template("dashboard.html",
pid=pid,
chart_type=chart_type,
myechart=Markup(jd_page.page.render_embed()))
def jd_charts(chart, pid):
product = get_product_by_pid(pid)
if product is None:
return '404!!!'
jd_page = JDPage(product)
chart_type = 1
if chart == 'bar':
jd_page.generate_stacked_bar_charts()
elif chart == 'pie':
jd_page.generate_pie_charts()
chart_type = 2
elif chart == 'wordcloud':
jd_page.generate_word_cloud_charts()
chart_type = 3
else:
return '404!'
return render_template("dashboard.html",
pid=pid,
chart_type=chart_type,
myechart=jd_page.page.render_embed(),
host=REMOTE_HOST,
script_list=jd_page.page.get_js_dependencies())
from util.db_util import get_product_by_pid
def get_summary_and_weight(comments):
"""
获得评论内容的摘要和权重
:param comments: 需要分析的评论列表
:return: 返回摘要列表和对应的权重列表
"""
d = {}
att = []
val = []
for c in comments:
nlp = SnowNLP(c.content)
# 评论获赞数越多，权重越高，取对数来平滑极差
w = int(math.log(c.votes + 1) + 1)**2
for kw in nlp.summary():
if d.get(kw):
d[kw] += w
else:
d[kw] = w
for k in d.keys():
att.append(k)
val.append(d.get(k))
return att, val
if __name__ == '__main__':
product = get_product_by_pid(100000822981)
def jd_dashboard(pid):
product = get_product_by_pid(pid)
if product is None:
return '404!!!'
return render_template('dashboard.html', product=product)