class HTMLExporter(object):
def __init__(self, path, user_id, chatroom_id, start_at, end_at):
self.wechat = WechatParser(path, user_id)
self.records = self.wechat.get_chatroom_records(chatroom_id, start_at, end_at)
self.friends = self.wechat.get_chatroom_friends(chatroom_id)
def export(self, export_path):
id_nicknames = {friend['id']: friend['nickname'] for friend in self.friends}
for record in self.records:
if not record['user_id']:
continue
record['nickname'] = id_nicknames.get(record['user_id']) or u'已退群'
env = Environment(loader=FileSystemLoader('we/contrib/html_exporter_res/'))
template = env.get_template('wechat.html')
output_from_parsed_template = template.render(records=self.records)
# make dir
export_full_path = os.path.realpath(export_path) + '/records'
os.makedirs(export_full_path)
# copy res
shutil.copytree('we/contrib/html_exporter_res/css', export_full_path+'/css')
shutil.copytree('we/contrib/html_exporter_res/img', export_full_path+'/img')
# build records html
with open(export_full_path+"/records.html", "w") as fh:
fh.write(output_from_parsed_template.encode('utf8'))
class HTMLExporter(object):
def __init__(self, path, user_id, chatroom_id, start_at, end_at):
self.wechat = WechatParser(path, user_id)
self.records = self.wechat.get_chatroom_records(chatroom_id, start_at, end_at)
self.friends = self.wechat.get_chatroom_friends(chatroom_id)
def export(self, export_path):
id_nicknames = {friend['id']: friend['nickname'] for friend in self.friends}
for record in self.records:
if not record['user_id']:
continue
record['nickname'] = id_nicknames.get(record['user_id']) or u'已退群'
searchpath = [os.path.join(path, 'we/contrib/html_exporter_res') for path in sys.path]
env = Environment(loader=FileSystemLoader(searchpath))
template = env.get_template('wechat.html')
output_from_parsed_template = template.render(records=self.records)
# make dir
export_full_path = os.path.join(os.path.realpath(export_path), 'records')
if not os.path.exists(export_full_path):
os.makedirs(export_full_path)
# copy res
# shutil.copytree('we/contrib/html_exporter_res/css', export_full_path+'/css')
# shutil.copytree('we/contrib/html_exporter_res/img', export_full_path+'/img')
if not os.path.exists(os.path.join(export_full_path, 'css')):
for path in searchpath:
source = os.path.join(path, 'css')
if os.path.exists(source):
# copy res
shutil.copytree(os.path.join(path, 'css'), os.path.join(export_full_path, 'css'))
shutil.copytree(os.path.join(path, 'img'), os.path.join(export_full_path, 'img'))
break
# build records html
with open(export_full_path + "/records.html", "w") as fh:
fh.write(output_from_parsed_template.encode('utf8'))
class ChatroomAnalytics(object):
def __init__(self, path, user_id, chatroom_id, start_at, end_at):
self.wechat = WechatParser(path, user_id)
self.records = self.wechat.get_chatroom_records(chatroom_id, start_at, end_at)
def get_stats(self, members=None):
# build stats for counters
counters = {
RecordType.SHORT_VIDEO: 0,
RecordType.LINK: 0,
RecordType.LOCATIOM: 0,
RecordType.EMOTION: 0,
RecordType.VIDEO: 0,
RecordType.CARD: 0,
RecordType.VOICE: 0,
RecordType.IMAGE: 0,
RecordType.TEXT: 0,
}
for record in self.records:
try:
counters[record['type']] += 1
except KeyError:
continue
# build stats for ranks
users_count = {}
for record in self.records:
if not record['user_id']:
continue
users_count[record['user_id']] = users_count.get(record['user_id'], 0) + 1
users_rank = sorted(users_count.items(), key=operator.itemgetter(1))
# build stats for silent users
member_ids = [member['id'] for member in members]
actice_users = users_count.keys()
silent_users = []
for user_id in member_ids:
if user_id not in actice_users:
silent_users.append(user_id)
return dict(counters=counters,
users_rank=users_rank,
silent_users=silent_users)
def get_user_stats(self, user_id):
pass
class FriendLabel(object):
def __init__(self, path, user_id):
self.wechat = WechatParser(path, user_id)
def get_stats(self):
friends = self.wechat.get_friends()
non_label_users = []
multi_label_users = []
# list friends without labels and with more than one label
for user in friends:
label_count = len(user['label_ids'])
if label_count == 0:
non_label_users.append(user)
elif label_count > 1:
multi_label_users.append(user)
return dict(non_label_users=non_label_users,
multi_label_users=multi_label_users)
class FriendLabel(object):
def __init__(self, path, user_id):
self.wechat = WechatParser(path, user_id)
def get_stats(self):
friends = self.wechat.get_friends()
non_label_users = []
multi_label_users = []
# list friends without labels and with more than one label
for user in friends:
label_count = len(user['label_ids'])
if label_count == 0:
non_label_users.append(user)
elif label_count > 1:
multi_label_users.append(user)
return dict(non_label_users=non_label_users, multi_label_users=multi_label_users)
def main():
logger = logging.getLogger('wechat_parser')
logger.addHandler(logging.StreamHandler())
logger.setLevel('DEBUG')
parser = argparse.ArgumentParser(prog='Wechat Explorer')
subparser = parser.add_subparsers(dest='command', help='sub-command help')
p1 = subparser.add_parser('list_friends', help='list all friends')
p1.add_argument('path', type=str)
p1.add_argument('user_id', type=str)
p2 = subparser.add_parser('list_chatrooms', help='list all chatrooms')
p2.add_argument('path', type=str)
p2.add_argument('user_id', type=str)
p3 = subparser.add_parser('get_chatroom_stats', help='get chatroom stats')
p3.add_argument('path', type=str)
p3.add_argument('user_id', type=str)
p3.add_argument('chatroom_id', type=str)
p3.add_argument('start_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p3.add_argument('end_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p4 = subparser.add_parser('get_chatroom_user_stats', help='get chatroom user stats')
p4.add_argument('path', type=str)
p4.add_argument('user_id', type=str)
p4.add_argument('chatroom_id', type=str)
p4.add_argument('chatroom_user_id', type=str)
p4.add_argument('start_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p4.add_argument('end_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5 = subparser.add_parser('export_chatroom_records', help='export chatroom records as HTML')
p5.add_argument('path', type=str)
p5.add_argument('user_id', type=str)
p5.add_argument('chatroom_id', type=str)
p5.add_argument('start_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('end_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('export_path', type=str)
p6 = subparser.add_parser('get_friend_label_stats', help='get friend label stats')
p6.add_argument('path', type=str)
p6.add_argument('user_id', type=str)
args = parser.parse_args()
if args.command == 'list_friends':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_friends()
for friend in friends:
msg = '%s %s(%s)' % (friend['id'], friend['nickname'], friend['remark'])
print msg.encode('utf-8')
elif args.command == 'list_chatrooms':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatrooms()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'get_chatroom_stats':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatroom_friends(args.chatroom_id)
id_nicknames = {friend['id']: friend['nickname'] for friend in friends}
from we.contrib.chatroom_analytics import ChatroomAnalytics
ca = ChatroomAnalytics(args.path, args.user_id, args.chatroom_id, args.start_at, args.end_at)
stats = ca.get_stats(friends)
counters = stats['counters']
for k, v in counters.items():
print '%s: %s' % (RecordTypeCN[k], v)
users_count = stats['users_rank']
print '\nTop 50 least talking:'
for i, user in enumerate(users_count[:50], 1):
print '%d %s: %s' % (i, id_nicknames.get(user[0]) or u'已退群', user[1])
print '\nTop 50 most talking:'
for i, user in enumerate(reversed(users_count[-50:]), 1):
print '%d %s: %s' % (i, id_nicknames.get(user[0]) or u'已退群', user[1])
silent_users = stats['silent_users']
print '\nSlient users:'
for user_id in silent_users:
print '%s %s' % (user_id, id_nicknames.get(user_id) or u'已退群')
elif args.command == 'get_chatroom_user_stats':
from we.contrib.chatroom_analytics import ChatroomAnalytics
stats = ChatroomAnalytics(args.path, args.user_id, args.chatroom_id, args.start_at, args.end_at)
print stats.get_user_stats(args.chatroom_user_id)
elif args.command == 'export_chatroom_records':
from we.contrib.html_exporter import HTMLExporter
exporter = HTMLExporter(args.path, args.user_id, args.chatroom_id, args.start_at, args.end_at)
exporter.export(args.export_path)
elif args.command == 'get_friend_label_stats':
wechat = WechatParser(args.path, args.user_id)
labels = wechat.get_labels()
from we.contrib.friend_label import FriendLabel
fl = FriendLabel(args.path, args.user_id)
stats = fl.get_stats()
print '\nUsers without labels:'
for user in stats['non_label_users']:
print '%s %s(%s)' % (user['id'], user['nickname'], user['remark'])
print '\nUsers with more than 2 labels:'
for user in stats['multi_label_users']:
label_str = ','.join([labels[label_id] for label_id in user['label_ids']])
print '%s %s(%s): %s' % (user['id'], user['nickname'], user['remark'], label_str)
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5 = subparser.add_parser('export_chatroom_records',
help='export chatroom records as HTML')
p5.add_argument('path', type=str)
p5.add_argument('user_id', type=str)
p5.add_argument('chatroom_id', type=str)
p5.add_argument('start_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('end_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('export_path', type=str)
args = parser.parse_args()
if args.command == 'list_friends':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_friends()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'list_chatrooms':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatrooms()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'get_chatroom_stats':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatroom_friends(args.chatroom_id)
id_nicknames = {friend['id']: friend['nickname'] for friend in friends}
def __init__(self, path, user_id, chatroom_id, start_at, end_at):
self.wechat = WechatParser(path, user_id)
self.records = self.wechat.get_chatroom_records(chatroom_id, start_at, end_at)
def main():
logger = logging.getLogger('wechat_parser')
logger.addHandler(logging.StreamHandler())
logger.setLevel('DEBUG')
parser = argparse.ArgumentParser(prog='Wechat Explorer')
subparser = parser.add_subparsers(dest='command', help='sub-command help')
p1 = subparser.add_parser('list_friends', help='list all friends')
p1.add_argument('path', type=str)
p1.add_argument('user_id', type=str)
p2 = subparser.add_parser('list_chatrooms', help='list all chatrooms')
p2.add_argument('path', type=str)
p2.add_argument('user_id', type=str)
p3 = subparser.add_parser('get_chatroom_stats', help='get chatroom stats')
p3.add_argument('path', type=str)
p3.add_argument('user_id', type=str)
p3.add_argument('chatroom_id', type=str)
p3.add_argument('start_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p3.add_argument('end_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p4 = subparser.add_parser('get_chatroom_user_stats',
help='get chatroom user stats')
p4.add_argument('path', type=str)
p4.add_argument('user_id', type=str)
p4.add_argument('chatroom_id', type=str)
p4.add_argument('chatroom_user_id', type=str)
p4.add_argument('start_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p4.add_argument('end_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5 = subparser.add_parser('export_chatroom_records',
help='export chatroom records as HTML')
p5.add_argument('path', type=str)
p5.add_argument('user_id', type=str)
p5.add_argument('chatroom_id', type=str)
p5.add_argument('start_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('end_at',
type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('export_path', type=str)
p6 = subparser.add_parser('get_friend_label_stats',
help='get friend label stats')
p6.add_argument('path', type=str)
p6.add_argument('user_id', type=str)
args = parser.parse_args()
if args.command == 'list_friends':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_friends()
for friend in friends:
msg = '%s %s(%s)' % (friend['id'], friend['nickname'],
friend['remark'])
print msg.encode('utf-8')
elif args.command == 'list_chatrooms':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatrooms()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'get_chatroom_stats':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatroom_friends(args.chatroom_id)
id_nicknames = {friend['id']: friend['nickname'] for friend in friends}
from we.contrib.chatroom_analytics import ChatroomAnalytics
ca = ChatroomAnalytics(args.path, args.user_id, args.chatroom_id,
args.start_at, args.end_at)
stats = ca.get_stats(friends)
counters = stats['counters']
for k, v in counters.items():
print '%s: %s' % (RecordTypeCN[k], v)
users_count = stats['users_rank']
print '\nTop 50 least talking:'
for i, user in enumerate(users_count[:50], 1):
print '%d %s: %s' % (i, id_nicknames.get(user[0])
or u'已退群', user[1])
print '\nTop 50 most talking:'
for i, user in enumerate(reversed(users_count[-50:]), 1):
print '%d %s: %s' % (i, id_nicknames.get(user[0])
or u'已退群', user[1])
silent_users = stats['silent_users']
print '\nSlient users:'
for user_id in silent_users:
print '%s %s' % (user_id, id_nicknames.get(user_id) or u'已退群')
elif args.command == 'get_chatroom_user_stats':
from we.contrib.chatroom_analytics import ChatroomAnalytics
stats = ChatroomAnalytics(args.path, args.user_id, args.chatroom_id,
args.start_at, args.end_at)
print stats.get_user_stats(args.chatroom_user_id)
elif args.command == 'export_chatroom_records':
from we.contrib.html_exporter import HTMLExporter
exporter = HTMLExporter(args.path, args.user_id, args.chatroom_id,
args.start_at, args.end_at)
exporter.export(args.export_path)
elif args.command == 'get_friend_label_stats':
wechat = WechatParser(args.path, args.user_id)
labels = wechat.get_labels()
from we.contrib.friend_label import FriendLabel
fl = FriendLabel(args.path, args.user_id)
stats = fl.get_stats()
print '\nUsers without labels:'
for user in stats['non_label_users']:
print '%s %s(%s)' % (user['id'], user['nickname'], user['remark'])
print '\nUsers with more than 2 labels:'
for user in stats['multi_label_users']:
label_str = ','.join(
[labels[label_id] for label_id in user['label_ids']])
print '%s %s(%s): %s' % (user['id'], user['nickname'],
user['remark'], label_str)
p4.add_argument('chatroom_id', type=str)
p4.add_argument('chatroom_user_id', type=str)
p4.add_argument('start_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p4.add_argument('end_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5 = subparser.add_parser('export_chatroom_records', help='export chatroom records as HTML')
p5.add_argument('path', type=str)
p5.add_argument('user_id', type=str)
p5.add_argument('chatroom_id', type=str)
p5.add_argument('start_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('end_at', type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'))
p5.add_argument('export_path', type=str)
args = parser.parse_args()
if args.command == 'list_friends':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_friends()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'list_chatrooms':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatrooms()
for friend in friends:
msg = '%s %s' % (friend['id'], friend['nickname'])
print msg.encode('utf-8')
elif args.command == 'get_chatroom_stats':
wechat = WechatParser(args.path, args.user_id)
friends = wechat.get_chatroom_friends(args.chatroom_id)
id_nicknames = {friend['id']: friend['nickname'] for friend in friends}
def __init__(self, path, user_id):
self.wechat = WechatParser(path, user_id)
def __init__(self, path, user_id, chatroom_id, start_at, end_at):
self.wechat = WechatParser(path, user_id)
self.records = self.wechat.get_chatroom_records(chatroom_id, start_at, end_at)
self.friends = self.wechat.get_chatroom_friends(chatroom_id)
def __init__(self, path, user_id):
self.wechat = WechatParser(path, user_id)