def main(pages):
mydb_beau = mydb()
mydict_beau = mydict()
mydb_beau.create_table('customer')
row_count = mydb_beau.get_count()
try:
for i in range(pages):
if i == 0:
url = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv3436&productId=100002795955&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'
else:
url = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv3423&productId=100002795955&score=0&sortType=5&page={}&pageSize=10&isShadowSku=0&rid=0&fold=1'.format(
i)
r_dict = get_page(url, headers=headers)
for element in r_dict['comments']:
customer_list = [
element['id'], element['content'].replace(" ", "").replace(
"\n", "").replace('"', "'")[0:255],
element['productColor'], element['creationTime']
]
mydb_beau.insert_data(customer_list)
if row_count == mydb_beau.get_count():
raise Already_exists()
else:
row_count = mydb_beau.get_count()
if i >= 10 and i % 10 == 0: #每十页休息30s
sleep(30)
except Already_exists:
pass
if i == pages - 1:
print('数据收集完成！')
else:
print('数据更新完成！')
list_kinds = ['天空之境', '亮黑色', '极光色', '赤茶橘', '珠光贝母']
color_list = mydb_beau.get_color_data()
list_numbers = [
color_list.count('天空之境'),
color_list.count('亮黑色'),
color_list.count('极光色'),
color_list.count('赤茶橘'),
color_list.count('珠光贝母')
]
for comment in mydb_beau.get_str_comments():
mydict_beau.add_into_comment_dictionary(
mydict_beau.divide_commint(comment))
if (mydict_beau.draw_dictionary(mydict_beau.sort_dictionary())):
print("高频词词云已经绘好！")
mydb_beau.close()
if (draw_color(list_kinds, list_numbers)):
print('颜色饼状图已经更新！')
def getHashNFileDict(directory):
hash_nfile_dict = mydict()
listOfFiles = getListOfFiles(directory)
for f in listOfFiles:
key = getHashSum(str(f).strip())
value = str(f).strip()
if hash_nfile_dict.has_key(key):
print 'key: %s and value: %s --Duplicate' % (key, value)
else:
hash_nfile_dict[key] = value
return hash_nfile_dict
def bb_sendmsg(user, addr, t):
''' send msg from site or client '''
jd = mydict(echoarea=t[0],
txt='\n'.join(t[5:]),
repto=t[3],
msgfrom=user,
addr=addr,
msgto=t[1],
title=t[2],
date=gts(),
accepted=gts())
save_msg(jd)
def save_msg(jd):
''' save bbmsg to db '''
#print jd
if not '.' in jd.echoarea:
return
msgid = hsh(bb_transform(jd, 60))
if check_title(jd.title):
add_to_topic(check_title(jd.title), msgid)
if '\n@@@@base64@@@@\n' in jd.txt:
nw = jd.txt.split('\n@@@@base64@@@@\n', 1)
jd.txt = nw[0] + '\n> spoiler!\n' + '\n'.join(
wrap(base64.b64encode(nw[1]), 50))
ji = mydict(mid=msgid, **jd)
msg.save(msg(**ji))
def app_rq_txt(s, data=mydict()):
''' convert db to plain text '''
kv = _mkkv(s)
out = '%s\n' % gts() if kv.appendts else ''
outrq = app_rq(s,data,url_req)
if kv.echolist:
if kv.cnt:
out += '\n'.join(['%s:%s:%s' % (n[0], n[1], n[0]) for n in outrq]) + '\n'
else:
out += '\n'.join(outrq) + '\n'
else:
out += outrq
if kv.endkey:
out += '\n%s' % kv.endkey
return out
def app_rq(s, data=mydict(), rf=get_pass):
''' application request, data = post data '''
kv = _mkkv(s)
if kv.echolist:
return echolist_pass(kv)
elif kv.point:
ea = data.ea or kv.ea or 'me'
msgto = data.msgto or kv.msgto or 'All'
title = data.title or kv.title or '***'
repto = data.repto or kv.repto
txt = data.outtxt or kv.txt
t = (ea, msgto, title, repto, txt.replace('\r\n', '\n'))
tb = '%s\n%s\n%s\n%s\n\n%s' % t
bb_sendmsg(data.uname, data.addr, tb.splitlines())
return 'msg ok:'
else:
return rf(kv)
def save_msg(jd):
''' save bbmsg to db '''
#print jd
if not '.' in jd.echoarea:
return
msgid = hsh(bb_transform(jd, 60))
if jd.repto:
obj = msg.get(msg.mid == jd.repto)
jd.topicid = obj.topicid
else:
jd.topicid = msgid
if '\n@@@@base64@@@@\n' in jd.txt:
nw = jd.txt.split('\n@@@@base64@@@@\n', 1)
jd.txt = nw[0] + '\n> spoiler!\n' + '\n'.join(
wrap(base64.b64encode(nw[1]), 50))
ji = mydict(mid=msgid, **jd)
msg.save(msg(**ji))
def point_msg(pauth, tmsg, uu):
if not uu:
return False
msgfrom = uu.uname.encode('utf-8')
tnew = tmsg.strip().replace('-', '+').replace('_', '/')
tlines = base64.b64decode(tnew)
lines = tlines.splitlines()
echoarea = lines[0]
mo = mydict(date=gts(),
accepted=gts(),
msgfrom=msgfrom,
addr='%s,%s' % (conf.STREET, uu.id),
echoarea=echoarea,
msgto=lines[1],
title=lines[2] or '***',
txt='\n'.join(lines[4:]))
if mo.txt.startswith('@repto:'):
tmpmsg = mo.txt.splitlines()
mo.repto = tmpmsg[0][7:]
mo.repto = mo.repto[:20]
mo.txt = '\n'.join(tmpmsg[1:])
save_msg(mo)
return 'msg ok:'
def _mkkv(s):
a = s.split('/')
if a[-1] == '':
a.pop()
return mydict(zip(a[0::2], a[1::2]))