def main():
redcli = redis.Redis(FLAGS.redishost, FLAGS.redisport)
key = "guang:rate:%s" % FLAGS.itemid
l = redcli.llen(key)
if l:
data = redcli.lrange(key, 0, l)
else:
data = []
logger.info("Total results: %s", l)
for d in data:
try:
comments = comments_pb2.comments()
comments.ParseFromString(d)
logger.info(
"data: %s %s %s %s %s %s %s %s",
comments.rateid,
comments.userid,
comments.result,
comments.time,
comments.userrank,
comments.userviplevel,
comments.user,
comments.content,
)
except:
logger.error(" failed %s %s - %s", d.encode("hex"), d, traceback.format_exc())
def taobao_comments_to_pb(self, items, page):
for s in items:
c = comments_pb2.comments()
c.user = s['user']['nick']
c.content = s['content']
try:
c.time = int(
time.mktime(
datetime.datetime.strptime(s['date'],
'%Y.%m.%d').timetuple()))
except:
c.time = int(
time.mktime(
datetime.datetime.strptime(
s['date'].encode('utf8'),
u'%Y\u5e74%m\u6708%d\u65e5 %H:%M'.encode(
'utf8')).timetuple()))
if s['rate'] == "1":
c.result = 1
elif s['rate'] == "0":
c.result = 2
elif s['rate'] == "-1":
c.result = 3
c.rateid = s['rateId']
try:
c.userid = int(s['user']['userId'])
except:
pass
c.userrank = s['user']['rank']
if s['user']['vipLevel']:
c.userviplevel = s['user']['vipLevel']
c.page = page
c.crawltime = int(time.time())
self.comments.append(c)
def taobao_comments_to_pb(self, items, page):
for s in items:
c = comments_pb2.comments()
c.user = s['user']['nick']
c.content = s['content']
try:
c.time = int(time.mktime(datetime.datetime.strptime(s['date'],'%Y.%m.%d').timetuple()))
except:
c.time = int(time.mktime(datetime.datetime.strptime(s['date'].encode('utf8'),u'%Y\u5e74%m\u6708%d\u65e5 %H:%M'.encode('utf8')).timetuple()))
if s['rate'] == "1":
c.result = 1
elif s['rate'] == "0":
c.result = 2
elif s['rate'] == "-1":
c.result = 3
c.rateid = s['rateId']
try:
c.userid = int(s['user']['userId'])
except:
pass
c.userrank = s['user']['rank']
if s['user']['vipLevel']:
c.userviplevel = s['user']['vipLevel']
c.page = page
c.crawltime = int(time.time())
self.comments.append(c)
def tmall_comments_to_pb(self, items):
for s in items:
c = comments_pb2.comments()
c.user = s['displayUserNick']
c.content = s['rateContent']
c.userviplevel = s['userVipLevel']
try:
c.userid = s['displayUserNumId']
except:
pass
c.time = int(time.mktime(datetime.datetime.strptime(s['rateDate'],'%Y-%m-%d %H:%M:%S').timetuple()))
c.rateid = s['id']
c.result = 1 #天猫无好中差评价，都给好评
self.comments.append(c)
def main():
redcli = redis.Redis(FLAGS.redishost, FLAGS.redisport)
key = "guang:rate:%s" % FLAGS.itemid
l = redcli.llen(key)
if l:
data = redcli.lrange(key, 0, l)
else:
data = []
logger.info("Total results: %s", l)
for d in data:
try:
comments = comments_pb2.comments()
comments.ParseFromString(d)
logger.info("data: %s %s %s %s %s %s %s %s", comments.rateid, comments.userid, comments.result, comments.time, comments.userrank, comments.userviplevel, comments.user, comments.content)
except:
logger.error(" failed %s %s - %s", d.encode('hex'), d, traceback.format_exc())
def tmall_comments_to_pb(self, items):
for s in items:
c = comments_pb2.comments()
c.user = s['displayUserNick']
c.content = s['rateContent']
c.userviplevel = s['userVipLevel']
try:
c.userid = s['displayUserNumId']
except:
pass
c.time = int(
time.mktime(
datetime.datetime.strptime(
s['rateDate'], '%Y-%m-%d %H:%M:%S').timetuple()))
c.rateid = s['id']
c.result = 1 #天猫无好中差评价，都给好评
self.comments.append(c)
def get_all():
db = get_db_engine()
items = db.execute("select id from item where status=1")
if items:
items_list = list(items)
pool = redis.ConnectionPool(host=FLAGS.redishost, port=FLAGS.redisport)
r = redis.Redis(connection_pool=pool)
for item in items_list:
itemid = item[0]
key = "guang:rate:%s" % itemid
l = r.llen(key)
if l:
data = r.lrange(key, 0, l)
else:
data = []
row = 0
good = 0
poor = 0
for d in data:
try:
coms = comments_pb2.comments()
coms.ParseFromString(d)
content = coms.content
result = coms.result
if len(content) >= FLAGS.char_limit:
row += 1
if result == 1:
good += 1
else:
poor += 1
except:
logger.error(" failed %s %s - %s", d.encode('hex'), d,
traceback.format_exc())
logger.info("%s data : %s %s %s", itemid, row, good, poor)
db.execute(
"replace into item_comments_total(item_id,row,good,poor) values (%s,%s,%s,%s) "
% (itemid, row, good, poor))
logger.info("total items : %s", len(items_list))
def get_all():
db = get_db_engine()
items = db.execute("select id from item where status=1")
if items:
items_list = list(items)
pool = redis.ConnectionPool(host=FLAGS.redishost, port=FLAGS.redisport)
r = redis.Redis(connection_pool=pool)
for item in items_list:
itemid = item[0]
key = "guang:rate:%s" % itemid
l = r.llen(key)
if l:
data = r.lrange(key, 0, l)
else:
data = []
row = 0
good = 0
poor = 0
for d in data:
try:
coms = comments_pb2.comments()
coms.ParseFromString(d)
content = coms.content
result = coms.result
if len(content) >= FLAGS.char_limit:
row += 1
if result == 1:
good += 1
else:
poor += 1
except:
logger.error(" failed %s %s - %s", d.encode("hex"), d, traceback.format_exc())
logger.info("%s data : %s %s %s", itemid, row, good, poor)
db.execute(
"replace into item_comments_total(item_id,row,good,poor) values (%s,%s,%s,%s) "
% (itemid, row, good, poor)
)
logger.info("total items : %s", len(items_list))
