def __add_location(self, msg, address, latitude, longitude, ts=None): location = model_im.Location() location.account_id = self.using_account.account_id location.address = address location.latitude = latitude location.longitude = longitude location.timestamp = ts self.model_im_col.db_insert_table_location(location) msg.location_id = location.location_id msg.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION
def parse_media_attachments(self, blob, src_msg): m = src_msg mp = MediaParser(blob, MediaParseDict) for ms in mp.res: t = ms.get('title') if t is not None and t is not '': msg = self.create_empty_message(m) msg.content = t msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT self.im.db_insert_table_message(msg) l = ms.get('photos') if l is not None: for pid in l: pname = self.get_photo_from_id(pid) if pname == "": continue msg = self.create_empty_message(m) msg.media_path = pname msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE self.im.db_insert_table_message(msg) if ms.__contains__('dphotos'): p = ms.get('dphotos') if p is None or p is '': pass else: msg = self.create_empty_message(m) pres = self.get_file_from_document(p) if pres is '': msg.content = 'document attachment image not cached...' msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT else: msg.media_path = pres msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE self.im.db_insert_table_message(msg) elif ms.__contains__('dvideo'): p = ms.get('dvideo') if p is None or p is '': pass else: msg = self.create_empty_message(m) pres = self.get_file_from_document(p) if pres is '': msg.content = 'document attachment video not cached...' msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT else: msg.media_path = pres msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO self.im.db_insert_table_message(msg) elif ms.__contains__('daudio'): p = ms.get('daudio') if p is None or p is '': pass else: msg =self.create_empty_message(m) pres = self.get_file_from_document(p) if pres is '': msg.content = 'document attahcment audio not cached...' msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT else: msg.media_path = pres msg.type = model_im.MESSAGE_CONTENT_TYPE_VOICE self.im.db_insert_table_message(msg) elif ms.__contains__('uri'): p = ms.get('uri') if p is None or p is '': pass else: msg = self.create_empty_message(m) msg.content = 'uri:%s' %ms.get('uri') msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT self.im.db_insert_table_message(msg) elif ms.__contains__('location'): p = ms.get('location') if p is None or len(p) == 0: pass else: m.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION locate = model_im.Location() locate.latitude = p.get('lati') locate.longitude = p.get('longti') locate.location_id = random.randint(0,0xffffffff) m.extra_id = locate.location_id self.im.db_insert_table_location(locate) if ms.__contains__('contact'): p = ms.get('contact') if p is None or len(p) == 0: pass else: m.type = model_im.MESSAGE_CONTENT_TYPE_CONTACT_CARD m.content = json.dumps(p) if ms.__contains__('video'): p = ms.get('video') if p is None or len(p) is 0: pass else: msg = self.create_empty_message(m) msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO msg.media_path = self.get_video(p.get('video_name')) self.im.db_insert_table_message(msg) if ms.__contains__('call'): p = ms.get('call') if p is None or len(p) is 0: pass else: src_msg.content = 'call id:{}, duration:{} second(s)'.format(p.get('call_id'), p.get('duration')) if ms.__contains__('deal'): p = ms.get('deal') if p is None or len(p) == 0: pass else : deal = model_im.Deal() deal.description = p.get('title') deal.money = p.get('money') src_msg.extra_id = deal.deal_id src_msg.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT src_msg.content = 'f' self.im.db_insert_table_deal(deal) if ms.__contains__('uimage'): p = ms.get('uimage') if len(p) == 0: pass else: for q in p: msg = self.create_empty_message(m) msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE msg.media_path = q self.im.db_insert_table_message(msg) if ms.__contains__('ulimage'): p = ms.get('ulimage') if len(p) == 0: pass else: for q in p: msg = self.create_empty_message(m) msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE msg.media_path = q self.im.db_insert_table_message(msg)
def get_group_messages(self): if self.read_file_path(): conn = System.Data.SQLite.SQLiteConnection( "Data Source = {0}; ReadOnly = True".format( self.source_path.PathWithMountPoint)) else: conn = System.Data.SQLite.SQLiteConnection( "Data Source = {0}".format(self.dest_path)) try: conn.Open() cmd = System.Data.SQLite.SQLiteCommand(conn) cmd.CommandText = """ select key_remote_jid as group_id, key_from_me, data, remote_resource, timestamp, media_url, media_mime_type, media_wa_type, media_name ,latitude ,longitude, status from messages where key_remote_jid in (select distinct(gjid) from group_participants a where jid is not null) """ reader = cmd.ExecuteReader() fs = self.root.FileSystem while reader.Read(): if canceller.IsCancellationRequested: return if GetString(reader, 0).find("broadcast") != -1: continue message = model_im.Message() message.source = self.source_path.AbsolutePath message.talker_type = 2 # 群聊天 message.account_id = self.account_id groups_id = GetString(reader, 0) message.is_sender = GetInt64(reader, 1) media_type = GetString(reader, 7) content = GetString(reader, 2) message.media_path = "" send_time = int(str(GetInt64(reader, 4))[:-3]) message.content = content message.talker_id = GetString(reader, 0) status = GetInt64(reader, 12) if groups_id in self.groups_dicts: message.talker_name = self.groups_dicts[groups_id] if GetInt64(reader, 1) == 1 and GetString(reader, 3) == "": message.sender_id = self.account_id message.sender_name = self.account_pushname elif GetInt64(reader, 1) == 1 and GetString(reader, 3) != "": message.sender_id = GetString(reader, 3) if GetString(reader, 3) in self.contacts_dicts: message.sender_name = self.contacts_dicts[GetString( reader, 3)] else: message.sender_name = GetString(reader, 3) elif GetInt64(reader, 1) == 0: sender_id = GetString(reader, 3) message.sender_id = sender_id if sender_id in self.contacts_dicts: message.sender_name = self.contacts_dicts[sender_id] else: message.sender_name = GetString(reader, 3) if status == 1: message.status = 1 else: message.status = 0 if media_type == '0': message.type = 1 elif media_type == '1': message.type = 2 if GetString(reader, 8): img_name = GetString(reader, 8) img_node = fs.Search(img_name) for i in img_node: img_path = i.AbsolutePath message.media_path = img_path elif media_type == '2': message.type = 3 if GetString(reader, 8): voice_name = GetString(reader, 8) voice_node = fs.Search(voice_name) for i in voice_node: voice_path = i.AbsolutePath message.media_path = voice_path elif media_type == '3': message.type = 4 if GetString(reader, 8): video_name = GetString(reader, 8) video_node = fs.Search(video_name) for i in video_node: video_path = i.AbsolutePath message.media_path = video_path elif media_type == '5': if GetFloat(reader, 9) > 0 and GetFloat(reader, 10) > 0: location = model_im.Location() message.type = 7 message.location_id = location.location_id lat, lng = unity_c37r.gcj2wgs_exact( GetFloat(reader, 9), GetFloat(reader, 10)) if lng: location.longitude = lng if lat: location.latitude = lat try: self.whatsapp.db_insert_table_location(location) except Exception as e: pass message.content = content message.send_time = send_time try: self.whatsapp.db_insert_table_message(message) except Exception as e: pass except Exception as e: TraceService.Trace(TraceLevel.Error, "{0}".format(e)) if conn != None: conn.Close() self.whatsapp.db_commit()
def parse_feed(self, node): '''解析动态数据''' print(self.comment_count) userid = re.findall('\d+', node.PathWithMountPoint)[-1] db = SQLiteParser.Database.FromNode(node, canceller) if db is None: return ts = SQLiteParser.TableSignature('moments') flag = 0 for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: if self._db_record_get_string_value(rec, '_id') == '': continue feed = model_im.Feed() #多媒体数据 media_url = self.parse_media_from_blob(rec, 'media_c') image_url = [] video_url = [] for url in media_url: if re.findall('images', url): image_url.append(url) if re.findall('videos', url): video_url.append(url) #获赞数 likescount = self._db_record_get_int_value(rec, 'likes_count_c') #动态id feedpk = self._db_record_get_string_value(rec, 'id_c') #评论数 comment_count = self.comment_count[feedpk] if feedpk in self.comment_count else 0 #动态时间 createtime = self._db_record_get_int_value(rec, 'createdTime_c') #动态位置 if not IsDBNull(rec['location_c'].Value): location = model_im.Location() feed.location_id = location.location_id # 地址ID[INT] loc = rec['location_c'].Value if not IsDBNull(loc): loc = Encoding.UTF8.GetString(loc) else: loc = '' location.address = self.illegal_char(loc) self.db_insert_table_location(location) #发送者 senderid = self._db_record_get_int_value(rec, 'owner_c') #动态文本 content = self._db_record_get_string_value(rec, 'value_c') feed.account_id = userid # 账号ID[TEXT] feed.sender_id = senderid # 发布者ID[TEXT] feed.content = content # 文本[TEXT] feed.image_path = ','.join(image_url) # 链接[TEXT] feed.video_path = ','.join(video_url) feed.send_time = createtime # 发布时间[INT] feed.likecount = likescount # 赞数量[INT] feed.commentcount = comment_count feed.comment_id = feedpk feed.source = node.AbsolutePath feed.deleted = rec.IsDeleted if feed.sender_id != 0: self.db_insert_table_feed(feed) except: pass self.db_commit()
def parse_message(self, node): '''解析消息数据''' try: db = SQLiteParser.Database.FromNode(node, canceller) userDir = node.PathWithMountPoint userid = re.findall('\d+', userDir)[-1] if db is None: return ts = SQLiteParser.TableSignature('messages') for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: message = Message() if canceller.IsCancellationRequested: break message.account_id = userid message.talker_id = self._db_record_get_int_value(rec, 'cid_c') otherUserId = self.conversation[message.talker_id] if message.talker_id in self.conversation else message.talker_id message.talker_name = self.friend[otherUserId] if otherUserId in self.friend else otherUserId message.sender_id = self._db_record_get_int_value(rec, 'owner_c') message.sender_name = self.friend[message.sender_id] if message.sender_id in self.friend else message.sender_id message.msg_id = self._db_record_get_string_value(rec, '_id') iscomment = 1 if not IsDBNull(rec['moment_c'].Value) else 0 content = self._db_record_get_string_value(rec, 'value_c') if not iscomment: if content == '[贴纸表情]': #贴纸表情 message.type = model_im.MESSAGE_CONTENT_TYPE_EMOJI sticker_index = '' if not IsDBNull(rec['api_only_accessory_c'].Value): value = rec['api_only_accessory_c'].Value start = 0 length = 0 for i, byte in enumerate(value): if int(byte) == 26: start = i+2 length = int(value[i+1]) break data = str(bytes(value)) sticker_index = data.encode('hex')[start*2:(start+length)*2:].decode('hex').decode('utf-8') if sticker_index != '': message.media_path = self.sticker[sticker_index] if sticker_index in self.sticker else '' else: message.content = content message.type = model_im.MEESAGE_CONTENT_TYPE_TEXT elif content == '[语音]': #语音消息 try: message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE media_url = self.parse_media_from_blob(rec, 'media_c') message.media_path = media_url[0] if len(media_url) != 0 else None if message.media_path == None: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') elif content == '[视频]': #视频 try: message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO media_url = self.parse_media_from_blob(rec, 'media_c') message.media_path = media_url[0] if len(media_url) != 0 else None if message.media_path == None: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') elif content == '[照片]': #照片 try: message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE media_url = self.parse_media_from_blob(rec, 'media_c') message.media_path = media_url[0] if len(media_url) != 0 else None if message.media_path == None: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'value_c') elif content == '[地理位置]': #地理位置 try: message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION location = model_im.Location() loc = rec['location_c'].Value if not IsDBNull(loc): loc = Encoding.UTF8.GetString(loc) else: loc = '' loc_address = self.illegal_char(loc) message.location_id = location.location_id location.address = loc_address self.db_insert_table_location(location) except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value(rec, 'location_c') else: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = content else: comments = model_im.FeedComment() comments.comment_id = self._db_record_get_string_value(rec, 'moment_c') # 评论ID[INT] if comments.comment_id not in self.comment_count: self.comment_count[comments.comment_id] = 1 else: self.comment_count[comments.comment_id] += 1 comments.sender_id = message.sender_id comments.sender_name = message.sender_name comments.content = content comments.create_time = self._get_timestamp(self._db_record_get_int_value(rec, 'createdTime_c')) comments.source = node.AbsolutePath comments.deleted = rec.IsDeleted self.db_insert_table_feed_comment(comments) message.send_time = self._get_timestamp(self._db_record_get_int_value(rec, 'createdTime_c')) message.source = node.AbsolutePath message.talker_type = model_im.CHAT_TYPE_FRIEND message.deleted = rec.IsDeleted if message.talker_id != 0 and message.type is not None: self.db_insert_table_message(message) except: traceback.print_exc() self.db_commit() except Exception as e: print(e)
def parse_message(self, dbPath, deleteFlag): '''解析消息数据''' db = SQLite.SQLiteConnection('Data Source = {}; ReadOnly = True'.format(dbPath)) db.Open() db_cmd = SQLite.SQLiteCommand(db) if deleteFlag is 1: self.db = SQLite.SQLiteConnection('Data Source = {}'.format(self.cachedb)) self.db.Open() self.db_cmd = SQLite.SQLiteCommand(self.db) fs = self.node.FileSystem try: db_cmd.CommandText = '''select distinct a.body, a.send_type, a.extra_uri, a._id, a.msg_date, a.participant_id, b.display_name, a.send_type, a.conversation_type, a.conversation_id, c.name, a.extra_mime, a.description, a.msg_info from messages as a left join participants_info as b on a.participant_id = b._id left join conversations as c on a.conversation_id = c._id''' sr = db_cmd.ExecuteReader() while (sr.Read()): try: message = model_im.Message() if canceller.IsCancellationRequested: break message.account_id = self.account_id message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.deleted = deleteFlag message.is_sender = self._db_reader_get_int_value(sr, 1) extra_mime = self._db_reader_get_int_value(sr, 11) description = self._db_reader_get_string_value(sr, 12) if extra_mime == 0: #文本 message.content = self._db_reader_get_string_value(sr, 0) message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT if message.content == '' and not IsDBNull(sr[13]): content = self._db_reader_get_string_value(sr, 13) text = json.loads(content) if content != '' else {} if 'pin' in text: if 'action' in text['pin']: if text['pin']['action'] == 'create': message.content = '创建群聊' message.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM elif 'pa_message_data' in text: if 'keyboard' in text['pa_message_data']: if 'Buttons' in text['pa_message_data']['keyboard']: values = text['pa_message_data']['keyboard']['Buttons'] content = [] for value in values: if 'ActionBody' in value and re.match('http', value['ActionBody']): content.append(value['ActionBody']) if 'Text' in value: content.append(re.sub('<[^>]*>', '', value['Text'])) if 'InternalBrowser' in value: content.append(value['InternalBrowser']['CustomTitle']) message.content = '\n'.join(content) elif extra_mime == 1: #图片 try: if not IsDBNull(sr[0]): message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE mediaPath = re.sub('.*/', '', sr[0]) for media_node in self.media_folders: if media_node is not None: nodes = media_node.Search(mediaPath) if len(list(nodes)) == 0: message.content = self._db_reader_get_string_value(sr, 12) + self._db_reader_get_string_value(sr, 0) messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT for node in nodes: message.media_path = node.AbsolutePath break else: message.content = self._db_reader_get_string_value(sr, 12) + self._db_reader_get_string_value(sr, 0) messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: pass elif extra_mime == 3: #视频 try: if not IsDBNull(sr[2]): message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO mediaPath = re.sub('.*/', '', sr[2]) nodes = fs.Search(mediaPath) if len(list(nodes)) == 0: message.content = mediaPath messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT for node in nodes: message.media_path = node.AbsolutePath break else: message.content = self._db_reader_get_string_value(sr, 12) + self._db_reader_get_string_value(sr, 0) messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: pass elif extra_mime == 4: #未知消息 try: message.content = '<表情包-未加载>' messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: pass elif extra_mime == 8: #链接 try: message.content = self._db_reader_get_string_value(sr, 0) message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT #if not IsDBNull(sr[0]): # dics = self._db_reader_get_string_value(sr, 0) # dics = dics.replace('\\', '').replace('[', '').replace(']', '').replace('},{', '}***{').split('***') # image_url = '' # push_text = '' # push_url = '' # for dic in dics: # dic = json.loads(dic) # if dic['Type'] == 'img': # image_url = dic['Action']['parameters']['url'] # elif dic['Type'] == 'info': # push_text = dic['PushText'] # push_url = dic['PreviewText'] # link = model_im.Link() # message.link_id = link.link_id # link.url = push_url # link.content = push_text # link.image = image_url # self.db_insert_table_link(link) # message.type = model_im.MESSAGE_CONTENT_TYPE_LINK #else: # message.content = self._db_reader_get_string_value(sr, 0) # messgae.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: traceback.print_exc() elif extra_mime == 9: #名片 try: content = self._db_reader_get_string_value(sr, 0) texts = content.replace('[', '').replace(']', '').replace('\n', '\\n').replace('},{', '}***{').split('***') contact_msg = '' for text in texts: dic = json.loads(text) if dic['Type'] == 'info': message.content = dic['PushText'] + '\n' + dic['PreviewText'] message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: pass elif extra_mime == 10: #附件 try: if not IsDBNull(sr[13]): content = self._db_reader_get_string_value(sr, 13) texts = json.loads(content) filename = texts['fileInfo']['FileName'] filesize = texts['fileInfo']['FileSize'] nodes = self.node.Parent.Parent.Search(filename + '$') if len(list(nodes)) == 0: message.content = '附件' + '\n' + '文件名:' + filename + '\n' + '文件大小:' + filesize message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT for node in nodes: message.media_path = node.AbsolutePath message.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT break else: message.content = '附件:' + self._db_reader_get_string_value(sr, 0) message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT except: pass elif extra_mime == 1000: #系统消息 message.content = self._db_reader_get_string_value(sr, 0).split('/')[0] message.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM elif extra_mime == 1002: #通话消息 message.content = self._db_reader_get_string_value(sr, 0) message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif extra_mime == 1006: #富文本消息 if not IsDBNull(sr[13]): text = sr[13].replace(r'\"', "'").replace("\'", '') if text.find('Latitude') != -1: latitude = re.findall('"Latitude":(.*?),', text)[0] longitude = re.findall('"Longitude":(.*?)}', text)[0] location = model_im.Location() message.location_id = location.location_id location.latitude = latitude location.longitude = longitude location.source = self.node.AbsolutePath self.db_insert_table_location(location) message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION else: message.content = text message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.msg_id = self._db_reader_get_int_value(sr, 3) message.send_time = self._get_timestamp(sr[4]) if not IsDBNull(sr[4]) else 0 message.sender_id = self._db_reader_get_int_value(sr, 5) message.sender_name = self._db_reader_get_string_value(sr, 6) message.source = self.node.AbsolutePath message.status = model_im.MESSAGE_STATUS_SENT if sr[7] == 1 or sr[7] == 0 else model_im.MESSAGE_STATUS_UNSENT message.talker_type = model_im.CHAT_TYPE_GROUP if sr[8] == 1 else model_im.CHAT_TYPE_OFFICIAL if sr[8] == 2 else model_im.CHAT_TYPE_FRIEND if sr[8] == 0 else model_im.CHAT_TYPE_SYSTEM message.talker_id = self._db_reader_get_int_value(sr, 9) message.talker_name = self._db_reader_get_string_value(sr, 10) if extra_mime in [0, 1, 3, 4, 8, 9, 10, 1000, 1002, 1006]: if self._isValid(message.content) or self._isValid(message.media_path) or message.location_id != 0 or message.link_id != 0: self.db_insert_table_message(message) except: traceback.print_exc() sr.Close() if deleteFlag is 0: self.db_commit() db_cmd.Dispose() db.Close() except Exception as e: print(e)
def get_group_messages(self, node, account_id): try: # results = [] db = SQLiteParser.Database.FromNode(node, canceller) if db is None: return if 'QOCMessage' not in db.Tables: return tbs = SQLiteParser.TableSignature("QOCMessage") for rec in db.ReadTableRecords(tbs, self.extractDeleted, True): if canceller.IsCancellationRequested: return try: messages = model_im.Message() messages.source = node.AbsolutePath if rec.Deleted == DeletedState.Deleted: messages.deleted = 1 messages.account_id = account_id messages.type = 2 # 群id if "sId" in rec and (not rec["sId"].IsDBNull): messages.talker_id = rec["sId"].Value # 1-系统 2-发送 3-receive messages.type = 2 # 发送者id if "fromId" in rec and (not rec["fromId"].IsDBNull): messages.sender_id = rec["fromId"].Value if rec["fromId"].Value == account_id: messages.is_sender = 1 # 消息id if "messageId" in rec and (not rec["messageId"].IsDBNull): messages.msg_id = rec["messageId"].Value if "type" in rec and ( not rec["type"].IsDBNull) and "content" in rec: message_type = rec["type"].Value json_data = "" # 文本 if message_type == 1: messages.type = MESSAGE_CONTENT_TYPE_TEXT messages.content = rec["content"].Value # 图片 if message_type == 2: try: messages.type = MESSAGE_CONTENT_TYPE_IMAGE json_data = json.loads(rec["content"].Value) if "url" in json_data: messages.media_path = json_data["url"] except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) # 位置 if message_type == 4: try: messages.type = MESSAGE_CONTENT_TYPE_LOCATION json_data = json.loads(rec["content"].Value) if "g" in json_data: loc = json_data["g"] try: lat, lng = loc.split(",") share_loc = model_im.Location() share_loc.source = node.AbsolutePath share_loc.latitude = float(lat) share_loc.longitude = float(lng) if "st" in rec and ( not rec["st"].IsDBNull): share_time = convert_to_unixtime( rec["st"].Value) messages.extra_id = share_loc.location_id self.qunar_db.db_insert_table_location( share_loc) except Exception as e: pass if "title" in json_data: messages.content = json_data["title"] if "imgurl" in json_data: messages.media_path = json_data["imgurl"] except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) # 链接 if message_type == 13: try: messages.type = MESSAGE_CONTENT_TYPE_LINK json_data = json.loads(rec["content"].Value) content = "" if "title" in json_data: content += json_data["title"] + "; " if "items" in json_data: for item in json_data["items"]: content += item["itemText"] + "; " if content is not None: messages.content = content except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) if message_type == 15: try: messages.type = MESSAGE_CONTENT_TYPE_LINK json_data = json.loads(rec["content"].Value) content = "" if "item" in json_data: for item in json_data["item"]: if "itemText" in item: content += item["itemText"] + "| " if content is not None: messages.content = content except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) if message_type == 16: try: messages.type = MESSAGE_CONTENT_TYPE_LINK json_data = json.loads(rec["content"].Value) content = "" if "items" in json_data: for item in json_data["items"]: content += item["itemText"] + "; " if content is not None: messages.content = content except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) if message_type == 19: try: messages.type = MESSAGE_CONTENT_TYPE_LINK json_data = json.loads(rec["content"].Value) content = "" if "price" in json_data: content += json_data["price"] + "| " if "status" in json_data: content += json_data["status"] + "| " if "products" in json_data: for item in json_data: if "body" in item: for i in item["body"]: for j in i: if "value" in j: content += j[ "value"] + "| " if "title" in item: for k in item["title"]: if "value" in j: content += j["value"] + "| " if content is not None: messages.content = content except Exception as e: TraceService.Trace( TraceLevel.Error, "{0},{1}".format(e, rec.Deleted)) else: messages.content = rec["content"].Value if "st" in rec and (not rec["st"].IsDBNull): messages.send_time = convert_to_unixtime( rec["st"].Value) if messages.account_id and messages.sender_id: self.qunar_db.db_insert_table_message(messages) # results.append(messages) except Exception as e: TraceService.Trace( TraceLevel.Info, "qunar(ios) get_group_messages record failed") # return results except Exception as e: TraceService.Trace(TraceLevel.Error, "qunar(ios) get_group_messages failed") # return [] self.qunar_db.db_commit()
def parse(self): for i in self.result_sql: current_id = None connection = sql.SQLiteConnection( 'Data Source = {}; ReadOnly=True'.format(i)) connection.Open() cmd = sql.SQLiteCommand(connection) cmd.CommandText = ''' select uid, nick, avatarMediaId, mobile, gender, birthdayValue, address, extension, email from contact ''' reader = cmd.ExecuteReader() idx = 0 f_dict = dict() while reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError("f****d") if idx == 0: a = model_im.Account() a.account_id = GetInt64(reader, 0) a.nickname = GetString(reader, 1) # real_name???? # photo ... pass for a while. self.get_picture(GetString(reader, 2), AVATAR) a.telephone = GetString(reader, 3) a.gender = GetInt64(reader, 4) a.birthday = GetInt64(reader, 5) a.address = GetString(reader, 6) a.email = GetString(reader, 8) r = GetString(reader, 7) if r is '': pass else: r = json.loads(r) if r.get('ownness') is not None: #r = r.get('ownness').get(status) pass else: pass self.im.db_insert_table_account(a) idx += 1 current_id = a.account_id continue f = model_im.Friend() f.friend_id = GetInt64(reader, 0) f.nickname = GetString(reader, 1) # photo.... # f.photo = #TODO generate photo id.... f.photo = self.get_picture(GetString(reader, 2), AVATAR) f.account_id = current_id f.telephone = GetString(reader, 3) f.gender = GetInt64(reader, 4) f.birthday = GetInt64(reader, 5) f.deleted = 0 f.email = GetString(reader, 8) f.address = GetString(reader, 6) f.type = model_im.FRIEND_TYPE_FRIEND r = GetString(reader, 7) if r is '': pass else: r = json.loads(r) if r.get('ownness') is not None: pass #TODO fix it later self.im.db_insert_table_friend(f) f_dict[f.friend_id] = f cmd.Dispose() cmd.CommandText = ''' select conversationId, title, createdAt, extensionJson, ownerId, memberLimit, memberCount, automaticIcon, customIcon from WKConversation where conversationType != 2 ''' reader = cmd.ExecuteReader() groups = list() while reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') g = model_im.Chatroom() g.account_id = current_id g.chatroom_id = GetString(reader, 0) g.name = GetString(reader, 1) g.photo = self.get_picture(GetString(reader, 8), AVATAR) g.create_time = GetInt64(reader, 2) / 1000 g.owner_id = GetInt64(reader, 4) g.max_member_count = GetInt64(reader, 5) g.member_count = GetInt64(reader, 6) groups.append(g.chatroom_id) self.im.db_insert_table_chatroom(g) chat_tbl_list = list() cmd.Dispose() cmd.CommandText = ''' select tbl_name from sqlite_master where tbl_name like 'WKChat_%' and type = 'table' ''' reader = cmd.ExecuteReader() while reader.Read(): r = GetString(reader, 0) if r is '' or r.__contains__('fts'): continue chat_tbl_list.append(r) cmd.Dispose() for r in chat_tbl_list: cmd.CommandText = ''' select messageId, conversationId, localSentTime, content, attachmentsType, senderId, attachmentsJson from {} '''.format(r) reader = cmd.ExecuteReader() while reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') msg = model_im.Message() msg.deleted = 0 msg.account_id = current_id cv_id = GetString(reader, 1) # update conversation id if cv_id.__contains__(':'): cvl = cv_id.split(':') if int(cvl[0]) == current_id: msg.talker_id = cvl[1] else: msg.talker_id = cvl[0] msg.talker_type = model_im.CHAT_TYPE_FRIEND else: msg.talker_id = cv_id msg.talker_type = model_im.CHAT_TYPE_GROUP msg.sender_id = GetInt64(reader, 5) msg.is_sender = 1 if msg.sender_id == current_id else 0 msg.content = GetString(reader, 3) msg.send_time = GetInt64(reader, 2) msg.msg_id = GetInt64(reader, 0) msg.source = i #TODO add other message decryptor.... and parse etc. tp = GetInt64(reader, 4) try: if tp == 1 or tp == 500 or tp == 501: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif tp == 1101: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT msg.content = '[call message]' elif tp == 600: msg.type = model_im.MESSAGE_CONTENT_TYPE_CONTACT_CARD string = GetString(reader, 6) js = json.loads(string) name = js.get('attachments')[0].get( 'extension').get('name') uid = js.get('attachments')[0].get( 'extension').get('uid') msg.content = 'uid:{}\nname:{}'.format(uid, name) elif tp == 102: msg.type = model_im.MESSAGE_CONTENT_TYPE_CONTACT_CARD string = GetString(reader, 6) js = json.loads(string) title = js.get('attachments')[0].get( 'extension').get('title') text = js.get('attachments')[0].get( 'extension').get('text') pic = js.get('attachments')[0].get( 'extension').get('picUrl') msg.content = 'title:{}\ntext:{}\npicUrl:{}'.format( title, text, pic) elif tp == 202 or tp == 103: msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO string = GetString(reader, 6) js = json.loads(string) media_id = js.get('attachments')[0].get( 'extension').get('picUrl') abs_path = self.root.PathWithMountPoint f_name = os.path.join( abs_path, "Library/Caches/videoCaches/%s.mp4" % media_id) if os.path.exists(f_name): msg.media_path = f_name else: msg.content = "video message: not cached" msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif tp == 104: # location ? string = GetString(reader, 6) js = json.loads(string) lati = js.get('attachments')[0].get( 'extension').get('latitude') lng = js.get('attachments')[0].get( 'extension').get('longitude') name = js.get('attachments')[0].get( 'extension').get('locationName') msg.location = md5(str(lati) + str(lng)) msg.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION msg.content = name l = model_im.Location() l.location_id = msg.location l.deleted = 0 l.latitude = lati l.longitude = lng l.address = name l.timestamp = GetInt64(reader, 2) self.im.db_insert_table_location(l) elif tp == 2: # image msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE string = GetString(reader, 6) js = json.loads(string) media_id = js.get('photoContent').get('mediaId') t_f_name = js.get('photoContent').get('filename') if t_f_name is None or t_f_name == "": #print(string) ext = '' else: fn, ext = os.path.splitext(t_f_name) if ext == "JPG" or ext == "jpg" or ext == 'MOV' or ext == 'mov': fn = self.get_picture(media_id, JPG) elif ext == 'PNG' or ext == 'png': fn = self.get_picture(media_id, PNG) elif ext == 'gif' or ext == 'GIF': fn = self.get_picture(media_id, GIF) else: fn = self.get_picture(media_id, SYSTEM) if fn != "": msg.media_path = fn else: msg.content = 'image message not cached...' msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT #TODO Fix it after rp is certain... elif tp == 900 or tp == 901: msg.content = GetString(reader, 6) msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT else: msg.content = GetString(reader, 3) if GetString( reader, 3) is not '' else GetString(reader, 6) msg.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM except: self.log_print('error occurs: {}'.format( GetString(reader, 3))) continue self.im.db_insert_table_message(msg) cmd.Dispose() # group members.... for g in groups: members = list() for t in chat_tbl_list: cmd.CommandText = ''' select senderId from {} where conversationId = '{}' group by senderId '''.format(t, g) reader = cmd.ExecuteReader() while reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') m_id = GetInt64(reader, 0) if members.__contains__(m_id) or m_id == current_id: continue members.append(m_id) cmd.Dispose() for m in members: cm = model_im.ChatroomMember() cm.account_id = current_id cm.member_id = m if f_dict.__contains__(m): #f = model_im.Friend() # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') f = f_dict[m] cm.address = f.address cm.display_name = f.nickname cm.birthday = f.birthday cm.photo = f.photo cm.deleted = 0 cm.gender = f.gender cm.email = f.email cm.telephone = f.telephone cm.signature = f.signature cm.chatroom_id = g self.im.db_insert_table_chatroom_member(cm) self.im.db_commit() #self.im.db_close() #以下部分不兼容老版本分析: cmd.CommandText = ''' select dingId, senderUid, sendAt, content from Ding ''' reader = cmd.ExecuteReader() feed_dict = dict() while reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') s_id = GetInt64(reader, 1) d_id = GetInt64(reader, 0) s_time = GetInt64(reader, 2) / 1000 s_content = GetString(reader, 3) feed = model_im.Feed() feed.sender_id = s_id feed.account_id = current_id feed.repeated = 0 feed.send_time = s_time feed.content = s_content feed_dict[d_id] = feed #feed.comments = s_id #self.im.db_insert_table_feed(feed) cmd.Dispose() cmd.CommandText = ''' select dingId, commentId, commenterUid, attachmentJSON, createAt from dingcomment ''' try: reader = cmd.ExecuteReader() except: reader = None while reader is not None and reader.Read(): # if canceller.IsCancellationRequested: # self.im.db_close() # raise IOError('f****d') fcm = model_im.FeedComment() d_id = GetInt64(reader, 0) if not feed_dict.__contains__(d_id): continue feed_dict[d_id].comments = str() feed_dict[d_id].comments += '{},'.format(GetInt64(reader, 1)) fcm.comment_id = GetInt64(reader, 1) string = GetString(reader, 3) try: fcm.content = json.loads(string).get('text') fcm.sender_id = GetInt64(reader, 1) fcm.sender_name = "" if not f_dict.__contains__( fcm.sender_id) else f_dict[fcm.sender_id].nickname fcm.create_time = GetInt64(reader, 4) / 1000 except: pass self.im.db_insert_table_feed_comment(fcm) for k in feed_dict: self.im.db_insert_table_feed(feed_dict[k]) feed_dict.clear() # no longer use... cmd.Dispose() # 出勤活动 cmd.CommandText = ''' select value from WebPersistenceModel where key like 'fastCheck_%' or 'asyncCheck_%' ''' try: reader = cmd.ExecuteReader() except: reader = None while reader is not None and reader.Read(): try: string = GetString(reader, 0) js = json.loads(string) feed = model_im.Feed() feed.sender_id = current_id addr = js.get('checkResult').get('address') time = js.get('checkResult').get('checkTime') f_id = js.get('checkResult').get('id') method = js.get('checkResult').get('locationMethod') s_type = js.get('checkResult').get('scheduleType') feed.content = ''' event:{} address:{} method:{} '''.format(s_type, addr, method) feed.send_time = time / 1000 self.im.db_insert_table_feed(feed) except: continue self.im.db_commit() cmd.Dispose() connection.Close() try: self.parse_recovery(i, current_id, chat_tbl_list) except: pass
def parse_friend(self): '''解析好友(联系人)数据''' try: db = SQLiteParser.Database.FromNode(self.node, canceller) if db is None: return ts = SQLiteParser.TableSignature('tb_near_contact') for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: friend = model_im.Friend() if canceller.IsCancellationRequested: break friend.account_id = self.account_id friend.deleted = rec.IsDeleted if 'fuserinfo' not in rec or IsDBNull( rec['fuserinfo'].Value): continue userinfo = json.loads(rec['fuserinfo'].Value.replace( '\\', '')) friend.friend_id = userinfo[ 'fuid'] if 'fuid' in userinfo.keys( ) else self._db_record_get_string_value(rec, 'fuid') friend.fullname = userinfo[ 'fnickname'] if 'fnickname' in userinfo.keys() else '' friend.nickname = userinfo['fnote'] if ( 'fnote' in userinfo.keys() and userinfo['fnote'] is not '') else friend.fullname friend.photo = userinfo[ 'ficon'] if 'ficon' in userinfo.keys() else None if 'fgender' in userinfo.keys(): friend.gender = 1 if userinfo['fgender'] == '1' else 0 lat = float(userinfo['flat']) / float(1000000) lng = float(userinfo['flng']) / float(1000000) location = model_im.Location() friend.location_id = location.location_id location.latitude = lat location.longitude = lng self.db_insert_table_location(location) friend.source = self.node.AbsolutePath friend.type = model_im.FRIEND_TYPE_RECENT self.db_insert_table_friend(friend) except: pass ts = SQLiteParser.TableSignature('tb_new_fans') for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: friend = model_im.Friend() if canceller.IsCancellationRequested: break friend.account_id = self.account_id friend.deleted = rec.IsDeleted if 'fuinfo' not in rec or IsDBNull(rec['fuinfo'].Value): continue userinfo = json.loads(rec['fuinfo'].Value.replace( '\\', '')) friend.friend_id = userinfo[ 'userid'] if 'userid' in userinfo.keys( ) else self._db_record_get_string_value(rec, 'fuid') friend.fullname = userinfo[ 'nickname'] if 'nickname' in userinfo.keys() else '' friend.nickname = friend.fullname friend.photo = userinfo['icon'] if 'icon' in userinfo.keys( ) else None if 'gender' in userinfo.keys(): friend.gender = 1 if userinfo['gender'] == 'm' else 0 lat = float(userinfo['lat']) / float(1000000) lng = float(userinfo['lng']) / float(1000000) location = model_im.Location() friend.location_id = location.location_id location.latitude = lat location.longitude = lng self.db_insert_table_location(location) friend.source = self.node.AbsolutePath friend.type = model_im.FRIEND_TYPE_FANS self.db_insert_table_friend(friend) except: pass self.db_commit() except Exception as e: print(e)
def get_groups_message(self, node, account_id): db = SQLiteParser.Database.FromNode(node, canceller) if db is None: return if 'session_table' not in db.Tables: return tbs = SQLiteParser.TableSignature("session_table") for rec in db.ReadTableRecords(tbs, True, False): if canceller.IsCancellationRequested: return try: # 群聊天 if "session_type" in rec and rec["session_type"].Value == 1: table_name = None talker_id = None talk_name = None if "session_id" in rec and ( not rec["session_id"].IsDBNull): talker_id = rec["session_id"].Value if rec["session_id"].Value in self.group_info.keys(): talk_name = self.group_info[ rec["session_id"].Value] if "name" in rec and (not rec["name"].IsDBNull): table_name = rec["name"].Value m_tbs = SQLiteParser.TableSignature(table_name) for m_rec in db.ReadTableRecords(m_tbs, True, False): try: if canceller.IsCancellationRequested: return message = model_im.Message() message.account_id = account_id if m_rec.Deleted == DeletedState.Deleted: message.deleted = 1 message.source = node.AbsolutePath message.talker_id = talker_id message.talker_name = talk_name # 默认都是好友聊天 message.talker_type = CHAT_TYPE_GROUP # 群聊天 # send_id if "msg_from_id" in m_rec and ( not m_rec["msg_from_id"].IsDBNull): message.sender_id = m_rec["msg_from_id"].Value if m_rec[ "msg_from_id"].Value in self.contact_list: message.sender_name = self.contact_list[ m_rec["msg_from_id"].Value] if m_rec["msg_from_id"].Value == account_id: message.is_sender = 1 if "msg_id" in m_rec and ( not m_rec["msg_id"].IsDBNull): message.msg_id = m_rec["msg_id"].Value if "msg_time" in m_rec and ( not m_rec["msg_time"].IsDBNull): message.send_time = convert_to_unixtime( m_rec["msg_time"].Value) # 判断消息类型 if "msg_type" in m_rec and ( not m_rec["msg_type"].IsDBNull): # text if m_rec["msg_type"].Value == 0: message.type = MESSAGE_CONTENT_TYPE_TEXT message.content = m_rec["msg_text"].Value # img elif m_rec["msg_type"].Value == 1: message.type = MESSAGE_CONTENT_TYPE_IMAGE try: data = json.loads( m_rec["msg_content"].Value) if "url" in data: message.media_path = data["url"] if "name" in data: message.content = data["name"] except Exception as e: pass # audio elif m_rec["msg_type"].Value == 2: message.type = MESSAGE_CONTENT_TYPE_VOICE try: data = json.loads( m_rec["msg_content"].Value) if "url" in data: message.media_path = data["url"] except Exception as e: pass message.content = m_rec["msg_text"].Value # video elif m_rec["msg_type"].Value == 3: message.type = MESSAGE_CONTENT_TYPE_VIDEO try: data = json.loads( m_rec["msg_content"].Value) if "url" in data: message.media_path = data["url"] except Exception as e: pass message.content = m_rec["msg_text"].Value # share location elif m_rec["msg_type"].Value == 4: message.type = MESSAGE_CONTENT_TYPE_LOCATION try: data = json.loads( m_rec["msg_content"].Value) loc = model_im.Location() if "lng" in data: loc.longitude = data["lng"] if "lat" in data: loc.latitude = data["lat"] if "title" in data: message.content = data["title"] loc.address = data["title"] self.bulletMessage.db_insert_table_location( loc) except Exception as e: pass # call 或者系统消息 elif m_rec["msg_type"].Value == 5: try: message.type = MESSAGE_CONTENT_TYPE_VOIP data = json.loads( m_rec["msg_content"].Value) if "duration" in data: message.content = "通话拨打时长 " + str( data["duration"]) + "s" else: message.type = MESSAGE_CONTENT_TYPE_SYSTEM if "data" in data: sysyem_data = data["data"] # self.get_chatroom_info(m_rec["msg_content"].Value) if "uinfos" in sysyem_data: if len(sysyem_data[ "uinfos"]) == 1: if "3" in sysyem_data[ "uinfos"][0]: message.content = sysyem_data[ "uinfos"][0][ "3"] + " 离开了群" elif len(sysyem_data[ "uinfos"]) == 2: if "3" in sysyem_data[ "uinfos"][0]: message.content = sysyem_data[ "uinfos"][0][ "3"] + " 进入了群" except Exception as e: pass # system messages elif m_rec["msg_type"].Value == 10: message.type = CHAT_TYPE_SYSTEM message.content = m_rec["msg_text"].Value # card elif m_rec["msg_type"].Value == 100 and m_rec[ "client_type"].Value == 1: message.type = MESSAGE_CONTENT_TYPE_CONTACT_CARD try: data = json.loads( m_rec["msg_content"].Value) if "data" in data: if "SMCardName" in data["data"]: message.content = data["data"][ "SMCardName"] except Exception as e: pass # 红包 elif m_rec["msg_type"].Value == 100 and ( not m_rec["push_content"].IsDBNull): message.type = MESSAGE_CONTENT_TYPE_RED_ENVELPOE message.content = m_rec[ "push_content"].Value if message.account_id and message.talker_id: self.bulletMessage.db_insert_table_message( message) except Exception as e: TraceService.Trace(TraceLevel.Info, e) except Exception as e: TraceService.Trace(TraceLevel.Info, e) self.bulletMessage.db_commit()
def convert_message_content(self, content, msg_obj): content = content.strip() if not (content.startswith("<") and content.endswith(">")): return content try: hp = PaHtmlParser() hp.feed(content) hp.close() tag_name = hp.first_dom.tag_name if tag_name == "deletemember": executor = hp.root['deletemember']['initiator'].data target = hp.root['deletemember']['target'].data return self.assemble_message_delete_member(executor, target) elif tag_name == "addmember": executor = hp.root['addmember']['initiator'].data target = hp.root['addmember']['target'].data return self.assemble_message_add_member(executor, target) elif tag_name == "ss": nodes = hp.root.get_all("ss") return self.assemble_message_emoji(nodes) elif tag_name == "uriobject": if hp.root['uriobject'].get("swift", None): encoded_info = hp.root['uriobject']['swift'].property[ 'b64'] decoded_info = TaoUtils.decode_base64(encoded_info) attachment = json.loads(decoded_info)["attachments"][0] url = attachment["content"]['images'][0]['url'] return url return hp.root['uriobject']['originalname'].property['v'] elif tag_name == 'partlist': return self.assemble_message_call(hp.root, msg_obj.account_id) elif tag_name == 'topicupdate': return self.assemble_message_rename_group(hp.root) elif tag_name == 'location': address = hp.root['location'].property['address'] latitude = float( hp.root['location'].property['latitude']) / 1000000 longitude = float( hp.root['location'].property['longitude']) / 1000000 ts = TimeHelper.convert_timestamp( hp.root['location'].property.get("timestamp", None)) location = model_im.Location() location.account_id = self.using_account.account_id location.address = address location.latitude = latitude location.longitude = longitude location.timestamp = ts self.model_im_col.db_insert_table_location(location) msg_obj.location_id = location.location_id msg_obj.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION return None elif tag_name == 'historydisclosedupdate': return self.assemble_message_history_closure(hp.root) elif tag_name == 'joiningenabledupdate': return self.assemble_message_close_add_memeber(hp.root) elif tag_name == "sms": return hp.root['sms']['defaults']['content'].data elif tag_name == "pictureupdate": return self.assemble_message_change_pic_bak(hp.root) else: return content except Exception as e: return content
def parse_feed(self, dbPath): '''解析动态数据''' db = SQLite.SQLiteConnection( 'Data Source = {}; ReadOnly = True'.format(dbPath)) db.Open() db_cmd = SQLite.SQLiteCommand(db) try: if self.db is None: return db_cmd.CommandText = '''select a.item_id, a.mediaBytesInfo, a.timestamp, a.user_id, group_concat(b."action") as "action", group_concat(b.comment) as comment, group_concat(b.timestamp) as comment_time, group_concat(b.user_id) as comment_user_id, group_concat(b.commend_id) as comment_id, a.deleted from bb_dl_item_info as a left join bb_dl_comment_info as b on a.item_id = b.item_id group by a.item_id''' sr = db_cmd.ExecuteReader() while (sr.Read()): try: feed = model_im.Feed() if canceller.IsCancellationRequested: break feed.account_id = self.account_id feed.sender_id = self._db_reader_get_int_value(sr, 3) feed.send_time = self._db_reader_get_int_value(sr, 2) location_value = '' #location_value用于保存提取出的位置信息的值 img_name = [] #img_name列表用于保存图片名称 text_content = '' #text_content用于保存提取出的正文内容 try: value = str(self._db_reader_get_blob_value( sr, 1)).encode('hex') data = [] #data用于存储格式化的字节信息 for i, item in enumerate(value): #生成data列表 if i % 2 == 0: data.append(value[i] + value[i + 1]) else: pass for i, d in enumerate(data): #判断与获取位置信息 if d == '42' and data[i + 1] == '4a': location_start = i + 3 location_lens = int(data[i + 2], 16) location_end = location_start + location_lens location_value = value[ location_start * 2:location_end * 2:].decode('hex').decode('utf-8') if d == '1a' and data[i + 2] == '0a': img_start = i + 4 img_lens = int(data[i + 3], 16) img_end = img_start + img_lens img_name.append( value[img_start * 2:img_end * 2:].decode('hex').decode('utf-8')) if d == '22': text_start = i + 2 text_lens = int(data[i + 1], 16) text_end = text_start + text_lens text_content = value[text_start * 2:text_end * 2:].decode('hex').decode( 'utf-8') if location_value is not '': location = model_im.Location() feed.location_id = location.location_id location.address = location_value self.db_insert_table_location(location) feed.content = text_content image_name = [] for image in img_name: nodes = self.cacheNode.Search('/' + image + '.*\..*$') for node in nodes: image_name.append(node.AbsolutePath) feed.image_path = ','.join(image_name) except: pass actions = self._db_reader_get_string_value(sr, 4).split(',') comment_id = self._db_reader_get_string_value(sr, 8).split(',') likes = [] comments = [] like_count = 0 comment_count = 0 for i, action in enumerate(actions): if action == 'like': like_count += 1 likes.append(comment_id[i]) elif not IsDBNull(action): comment_count += 1 comments.append(comment_id[i]) feed.likes = ','.join(likes) feed.likecount = like_count feed.comments = ','.join(comment_id) feed.commentcount = comment_count feed.deleted = self._db_reader_get_int_value(sr, 9) if feed.content is not None and feed.image_path is not None: self.db_insert_table_feed(feed) except: traceback.print_exc() sr.Close() self.db_commit() db_cmd.Dispose() db.Close() except Exception as e: print(e)
def get_messages(self, node, account_id): m_node = node.GetByPath("msg.db") if m_node is None: return try: db = SQLiteParser.Database.FromNode(m_node, canceller) if db is None: return if 'msghistory' not in db.Tables: return tbs = SQLiteParser.TableSignature("msghistory") for rec in db.ReadTableRecords(tbs, self.extractDeleted, True): try: if canceller.IsCancellationRequested: return messages = model_im.Message() messages.account_id = account_id messages.source = m_node.AbsolutePath if rec.Deleted == DeletedState.Deleted: messages.deleted = 1 messages.type = 1 # 文本 if "id" in rec and (not rec["id"].IsDBNull): messages.talker_id = rec["id"].Value if rec["id"].Value in self.friend_list.keys(): messages.talker_type = CHAT_TYPE_FRIEND messages.talker_name = self.friend_list[ rec["id"].Value] if rec["id"].Value in self.group_list.keys(): messages.talker_type = CHAT_TYPE_GROUP messages.talker_name = self.group_list[ rec["id"].Value] if "fromid" in rec and (not rec["fromid"].IsDBNull): messages.sender_id = rec["fromid"].Value if rec["fromid"].Value in self.all_send_nick_name: messages.sender_name = self.all_send_nick_name[ rec["fromid"].Value] if rec["fromid"].Value == account_id: messages.is_sender = 1 if "time" in rec and (not rec["time"].IsDBNull): messages.send_time = convert_to_unixtime( rec["time"].Value) # 判断消息类型 if "msgtype" in rec and (not rec["msgtype"].IsDBNull): # text if rec["msgtype"].Value == 0: messages.type = MESSAGE_CONTENT_TYPE_TEXT messages.content = rec["content"].Value # img elif rec["msgtype"].Value == 1: messages.type = MESSAGE_CONTENT_TYPE_IMAGE try: data = json.loads(rec["attach"].Value) if "url" in data: messages.media_path = data["url"] if "name" in data: messages.content = data["name"] except Exception as e: pass # audio elif rec["msgtype"].Value == 2: messages.type = MESSAGE_CONTENT_TYPE_VOICE try: data = json.loads(rec["attach"].Value) if "url" in data: messages.media_path = data["url"] except Exception as e: pass messages.content = rec["content"].Value # video elif rec["msgtype"].Value == 3: messages.type = MESSAGE_CONTENT_TYPE_VIDEO try: data = json.loads(rec["attach"].Value) if "url" in data: messages.media_path = data["url"] except Exception as e: pass messages.content = rec["content"].Value # share location elif rec["msgtype"].Value == 4: messages.type = MESSAGE_CONTENT_TYPE_LOCATION try: data = json.loads(rec["attach"].Value) loc = model_im.Location() if "lng" in data: loc.longitude = data["lng"] if "lat" in data: loc.latitude = data["lat"] if "title" in data: messages.content = data["title"] loc.address = data["title"] self.bulletMessage.db_insert_table_location( loc) except Exception as e: pass # call 或者系统消息 elif rec["msgtype"].Value == 5: try: messages.type = MESSAGE_CONTENT_TYPE_VOIP data = json.loads(rec["attach"].Value) if "duration" in data: messages.content = "通话拨打时长 " + str( data["duration"]) + "s" else: messages.type = MESSAGE_CONTENT_TYPE_SYSTEM if "data" in data: sysyem_data = data["data"] self.get_chatroom_info( rec["attach"].Value) if "uinfos" in sysyem_data: if len(sysyem_data["uinfos"]) == 1: if "3" in sysyem_data[ "uinfos"][0]: messages.content = sysyem_data[ "uinfos"][0][ "3"] + " 离开了群" elif len(sysyem_data["uinfos"] ) == 2: if "3" in sysyem_data[ "uinfos"][0]: messages.content = sysyem_data[ "uinfos"][0][ "3"] + " 进入了群" except Exception as e: pass # system messages elif rec["msgtype"].Value == 10: messages.type = CHAT_TYPE_SYSTEM messages.content = rec["content"].Value # card elif rec["msgtype"].Value == 100 and rec[ "fromclient"].Value == 1: messages.type = MESSAGE_CONTENT_TYPE_CONTACT_CARD try: data = json.loads(rec["attach"].Value) if "data" in data: if "SMCardName" in data["data"]: messages.content = data["data"][ "SMCardName"] except Exception as e: pass # 红包 elif rec["msgtype"].Value == 100 and ( not rec["push"].IsDBNull): messages.type = MESSAGE_CONTENT_TYPE_RED_ENVELPOE messages.content = rec["push"].Value if messages.account_id and messages.talker_id: self.bulletMessage.db_insert_table_message( messages) except Exception as e: print(e) except Exception as e: print(e) self.bulletMessage.db_commit()
def parse_message(self, dbPath): '''解析消息数据''' db = SQLite.SQLiteConnection( 'Data Source = {}; ReadOnly = True'.format(dbPath)) db.Open() db_cmd = SQLite.SQLiteCommand(db) fs = self.node.FileSystem try: if db is None: return db_cmd.CommandText = '''select a.time, a.otheruid, a.msgid, a.msgstate, a.attachment, a.distance, a.myuid, a.content, a.msgtype, a.deleted, b.nickname from DBTableName_P2PMsg as a left join DBTableName_UserRelation as b on a.otheruid = b.otheruid''' sr = db_cmd.ExecuteReader() while (sr.Read()): try: message = model_im.Message() if canceller.IsCancellationRequested: break message.account_id = self.account_id message.deleted = self._db_reader_get_int_value(sr, 9) message.send_time = self._get_timestamp( self._db_reader_get_int_value(sr, 0)) message.talker_id = self._db_reader_get_int_value(sr, 1) message.talker_name = self._db_reader_get_string_value( sr, 10) msgstate = self._db_reader_get_int_value(sr, 3) message.sender_id = self._db_reader_get_int_value( sr, 1 ) if msgstate == 1 else self.account_id if msgstate == 2 else 0 message.sender_name = self._db_reader_get_string_value( sr, 10 ) if msgstate == 1 else self.account_name if msgstate == 2 else '未知用户' message.is_sender = 1 if msgstate == 2 else 0 message.msgid = self._db_reader_get_int_value(sr, 2) message.talker_type = model_im.CHAT_TYPE_FRIEND msgtype = self._db_reader_get_int_value(sr, 8) if msgtype == 1: #普通文本 message.content = self._db_reader_get_string_value( sr, 7) message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif msgtype == 2: #jpg message.media_path = self._db_reader_get_string_value( sr, 4) message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif msgtype == 3: #mp3 media_name = self._db_reader_get_string_value(sr, 4) message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE nodes = fs.Search(media_name) for node in nodes: message.media_path = node.AbsolutePath break elif msgtype == 4: #3gp message.media_path = self._db_reader_get_string_value( sr, 4) message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO elif msgtype == 5: #地理位置 location = model_im.Location() loc = self._db_reader_get_string_value(sr, 4) loc = loc.split(',') latitude = float(loc[0]) / 1000000 longitude = float(loc[1]) / 1000000 message.location_id = location.location_id message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION location.latitude = latitude location.longitude = longitude location.address = self._db_reader_get_string_value( sr, 7) self.db_insert_table_location(location) elif msgtype == 6: #png message.media_path = self._db_reader_get_string_value( sr, 4) message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE self.db_insert_table_message(message) except: traceback.print_exc() sr.Close() self.db_commit() db_cmd.Dispose() db.Close() except Exception as e: print(e)
def parse_message(self, node): '''解析消息数据''' try: db = SQLiteParser.Database.FromNode(node, canceller) userDir = node.PathWithMountPoint userid = re.findall('\d+', userDir)[-1] if db is None: return ts = SQLiteParser.TableSignature('Message') for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: message = Message() if canceller.IsCancellationRequested: break message.account_id = userid message.talker_id = self._db_record_get_int_value( rec, 'conversationID') otherUserId = self.conversation[ message. talker_id] if message.talker_id in self.conversation else message.talker_id message.talker_name = self.friend[ otherUserId] if otherUserId in self.friend else otherUserId message.sender_id = self._db_record_get_int_value( rec, 'owner_primaryKeyID') message.sender_name = self.friend[ message. sender_id] if message.sender_id in self.friend else message.sender_id message.msg_id = self._db_record_get_string_value( rec, 'objectID') content_type = self._db_record_get_int_value( rec, 'type' ) #0:文本消息1:图片2:视频3:语音消息4:表情包5:私密真心话(文本消息)6:地理位置系统消息 if content_type == 0: #文本消息 message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 1: #图片消息 try: message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE local_dic = self._db_record_get_blob_value( rec, 'pendingMediaDictionary') tree = BPReader.GetTree( MemoryRange.FromBytes( rec['mediaDictionaries'].Value)) if tree is None: break image_json = self.bplist2json(tree) if local_dic is not None: local_info = json.loads(local_dic) media_name = local_info[ 'CacheImagePathKey'] if 'CacheImagePathKey' in local_info else '' media_nodes = self.node.Search(media_name + '$') if len(list(media_nodes)) != 0: message.media_path = list( media_nodes)[0].AbsolutePath else: message.media_path = image_json['key'][ 'url'] else: message.media_path = image_json['key']['url'] except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 2: #视频 try: message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO local_dic = self._db_record_get_blob_value( rec, 'mediaDatas') tree = BPReader.GetTree( MemoryRange.FromBytes( rec['mediaDictionaries'].Value)) if tree is None: break video_json = self.bplist2json(tree) if local_dic is not None: local_info = json.loads(local_dic) media_name = local_info[ 'url'] if 'url' in local_info else '' media_nodes = self.node.Search(media_name + '$') if len(list(media_nodes)) != 0: message.media_path = list( media_nodes)[0].AbsolutePath else: message.media_path = audio_json['key'][ 'url'] else: message.media_path = audio_json['key']['url'] except: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 3: #语音 try: message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE local_dic = self._db_record_get_blob_value( rec, 'pendingMediaDictionary') tree = BPReader.GetTree( MemoryRange.FromBytes( rec['mediaDictionaries'].Value)) if tree is None: break audio_json = self.bplist2json(tree) if local_dic is not None: local_info = json.loads(local_dic) media_name = local_info[ 'url'] if 'url' in local_info else '' media_type = local_info[ 'mediaType'] if 'mediaType' in local_info else '' duration = local_info[ 'duration'] if 'duration' in local_info else '' media_nodes = self.node.Search(media_name + '$') if len(list(media_nodes)) != 0: message.media_path = list( media_nodes)[0].AbsolutePath else: message.media_path = audio_json['key'][ 'url'] else: message.media_path = audio_json['key']['url'] except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 4: #表情包 try: message.type = model_im.MESSAGE_CONTENT_TYPE_EMOJI sticker_pk = self._db_record_get_string_value( rec, 'sticker_primaryKeyID') sticker_cnt = self.sticker[sticker_pk] sticker_cnt = json.loads(sticker_cnt) message.media_path = sticker_cnt['url'] except: traceback.print_exc() message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 5: #私密真心话 message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 6: #地理位置 try: message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION location = model_im.Location() location_info = self._db_record_get_blob_value( rec, 'locationDictionary') location_info = json.loads(location_info) message.location_id = location.location_id location.latitude = location_info['coordinates'][0] location.longitude = location_info['coordinates'][ 1] location.address = location_info[ 'address'] + location_info['name'] self.db_insert_table_location(location) except: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 7: #撤回消息 message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.content = self._db_record_get_string_value( rec, 'value') elif content_type == 9: #转发动态 message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT message.moment_id = self._db_record_get_int_value( rec, 'momentID') message.content = "[转发动态]" + str(message.moment_id) message.send_time = self._get_timestamp( self._db_record_get_int_value(rec, 'createdTime')) message.talker_type = model_im.CHAT_TYPE_FRIEND message.deleted = rec.IsDeleted if message.talker_id != 0: self.db_insert_table_message(message) except: pass self.db_commit() except Exception as e: print(e)
def parse_message(self): '''解析消息数据''' fs = self.node.FileSystem try: db = SQLiteParser.Database.FromNode(self.node, canceller) if db is None: return ts = SQLiteParser.TableSignature('tb_personal_message') for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: message = model_im.Message() if canceller.IsCancellationRequested: break message.account_id = self.account_id message.deleted = rec.IsDeleted message.send_time = self._get_timestamp( self._db_record_get_int_value(rec, 'timestamp')) message.talker_id = self._db_record_get_int_value( rec, 'f_uid') if not 'content' in rec or IsDBNull(rec['content'].Value): continue content = json.loads(rec['content'].Value) if 'user' in content.keys(): userinfo = content['user'] userid = userinfo[ 'userid'] if 'userid' in userinfo.keys() else None nickname = userinfo[ 'nickname'] if 'nickname' in userinfo.keys( ) else '' message.talker_name = nickname # birthday = userinfo['birthday'] if 'birthday' in userinfo.keys() else 0 # gender = userinfo['gender'] if 'gender' in userinfo.keys() else 0 # lng = userinfo['lng'] if 'lng' in userinfo.keys() else 0 # note = userinfo['notes'] if 'notes' in userinfo.keys() else '' # age = userinfo['age'] if 'age' in userinfo.keys() else 0 # icon = userinfo['icon'] if 'icon' in userinfo.keys() else None # lat = userinfo['lat'] if 'lat' in userinfo.keys() else 0 sendtype = self._db_record_get_int_value(rec, 'sendtype') message.sender_id = self._db_record_get_int_value( rec, 'f_uid' ) if sendtype == 2 else self.account_id if sendtype == 1 else 0 message.sender_name = message.talker_name if sendtype == 2 else self.account_name if sendtype == 1 else '' message.is_sender = 1 if sendtype == 1 else 0 message.msgid = self._db_record_get_int_value( rec, 'server_id') message.talker_type = model_im.CHAT_TYPE_FRIEND msgtype = self._db_record_get_int_value( rec, 'message_type') if msgtype == 1: #普通文本 message.content = content[ 'content'] if 'content' in content.keys() else '' message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif msgtype == 2: #jpg message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif msgtype == 3: #mp3 media_name = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE if media_name is not None: pardir = os.path.basename( os.path.dirname(media_name)) filename = os.path.basename(media_name) nodes = fs.Search('/' + os.path.join( pardir, filename).replace('\\', '/') + '$') if nodes is None: message.content = '音频消息:' + filename message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT for node in nodes: message.media_path = node.AbsolutePath break elif msgtype == 4: #3gp message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO elif msgtype == 5: #地理位置 location = model_im.Location() loc = content[ 'attachment'] if 'attachment' in content.keys( ) else None if loc is not None: loc = loc.split(',') longitude = float(loc[0]) / 1000000 latitude = float(loc[1]) / 1000000 message.location_id = location.location_id message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION location.latitude = latitude location.longitude = longitude location.address = content[ 'content'] if 'content' in content.keys() else None location.timestamp = content[ 'datetime'] if 'datetime' in content.keys() else 0 self.db_insert_table_location(location) elif msgtype == 6: #png message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif msgtype == 9: #表情包 message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE self.db_insert_table_message(message) except: pass self.db_commit() ts = SQLiteParser.TableSignature('tb_group_message') dic = {} self.member_count = {} for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: message = model_im.Message() if canceller.IsCancellationRequested: break message.account_id = self.account_id message.deleted = rec.IsDeleted message.send_time = self._get_timestamp( self._db_record_get_int_value(rec, 'timestamp')) message.talker_id = self._db_record_get_int_value( rec, 'groupid') if not 'content' in rec or IsDBNull(rec['content'].Value): continue msginfo = json.loads(rec['content'].Value) if 'user' in msginfo.keys(): #群好友信息 userinfo = msginfo['user'] userid = userinfo[ 'userid'] if 'userid' in userinfo.keys() else None if userid == self._db_record_get_int_value( rec, 'userid'): message.is_sender = 1 else: message.is_sender = 0 message.sender_id = userid nickname = userinfo[ 'nickname'] if 'nickname' in userinfo.keys( ) else '' message.sender_name = nickname birthday = userinfo[ 'birthday'] if 'birthday' in userinfo.keys() else 0 gender = userinfo[ 'gender'] if 'gender' in userinfo.keys() else 0 lng = userinfo['lng'] if 'lng' in userinfo.keys( ) else 0 age = userinfo['age'] if 'age' in userinfo.keys( ) else 0 icon = userinfo['icon'] if 'icon' in userinfo.keys( ) else None lat = userinfo['lat'] if 'lat' in userinfo.keys( ) else 0 friend = model_im.Friend() member = model_im.ChatroomMember() flag = 0 friend.account_id = self.account_id member.account_id = self.account_id friend.friend_id = userid member.member_id = userid member.chatroom_id = message.talker_id friend.fullname = nickname member.display_name = nickname friend.nickname = nickname friend.photo = icon member.photo = icon friend.gender = 1 if gender == 'm' else 0 member.gender = friend.gender friend.age = age member.age = age friend.source = self.node.AbsolutePath member.source = self.node.AbsolutePath location = model_im.Location() friend.location_id = location.location_id location.latitude = float(lat) / float(1000000) location.longitude = float(lng) / float(1000000) friend.type = model_im.FRIEND_TYPE_GROUP_FRIEND if not userid in dic.keys(): dic[userid] = [rec.IsDeleted] elif rec.IsDeleted not in dic[userid]: dic[userid].append(rec.IsDeleted) else: flag = 1 if flag == 0: self.db_insert_table_location(location) self.db_insert_table_friend(friend) self.db_insert_table_chatroom_member(member) if member.chatroom_id not in self.member_count.keys( ): self.member_count[member.chatroom_id] = 1 else: self.member_count[member.chatroom_id] += 1 message.msg_id = self._db_record_get_int_value( rec, 'messageid') msgtype = self._db_record_get_int_value( rec, 'message_type') content = json.loads( self._db_record_get_string_value(rec, 'content')) if msgtype == 1: #普通文本 message.content = content[ 'content'] if 'content' in content.keys() else '' message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif msgtype == 2: #jpg message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif msgtype == 3: #mp3 media_name = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE if media_name is not None: pardir = os.path.basename( os.path.dirname(media_name)) filename = os.path.basename(media_name) nodes = fs.Search('/' + os.path.join( pardir, filename).replace('\\', '/') + '$') if nodes is None: message.content = '音频消息:' + filename message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT for node in nodes: message.media_path = node.AbsolutePath break elif msgtype == 4: #3gp message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO elif msgtype == 5: #地理位置 location = model_im.Location() loc = content[ 'attachment'] if 'attachment' in content.keys( ) else None if loc is not None: loc = loc.split(',') latitude = float(loc[0]) / 1000000 longitude = float(loc[1]) / 1000000 message.location_id = location.location_id message.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION location.latitude = latitude location.longitude = longitude location.address = content[ 'content'] if 'content' in content.keys() else None location.timestamp = content[ 'datetime'] if 'datetime' in content.keys() else 0 self.db_insert_table_location(location) elif msgtype == 6: #png message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif msgtype == 9: #表情包 message.media_path = content['attachment'].replace( '\\', '') if 'attachment' in content.keys() else None message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE message.talker_type = model_im.CHAT_TYPE_GROUP self.db_insert_table_message(message) except: pass #print(self.member_count) self.db_commit() except Exception as e: print(e)
def parse_feed(self, node): '''解析动态数据''' userid = re.findall('\d+', node.PathWithMountPoint)[-1] db = SQLiteParser.Database.FromNode(node, canceller) if db is None: return try: ts = SQLiteParser.TableSignature('Moment') self.sticker = {} for rec in db.ReadTableRecords(ts, self.extractDeleted, True): try: if self._db_record_get_string_value(rec, 'objectID') == '': continue feed = model_im.Feed() #多媒体数据 media_url = '' tree = BPReader.GetTree( MemoryRange.FromBytes(rec['mediaDictionaries'].Value)) if tree is None: break media_json = self.bplist2json(tree) if media_json is not None: media_url = media_json['key']['url'] #获赞数 likescount = self._db_record_get_int_value( rec, 'likesCount') #评论数 commentcount = self._db_record_get_int_value( rec, 'commentsCount') #动态id feedpk = self._db_record_get_int_value(rec, 'primaryKeyID') #动态时间 createtime = self._db_record_get_int_value( rec, 'createdTime') #动态位置 location = self._db_record_get_blob_value( rec, 'locationDictionary') if location is not None: location = json.loads(location) #location = Encoding.UTF8.GetString(rec['locationDictionary'].Value) if not IsDBNull(rec['locationDictionary'].Value) else None if location is not None: try: coordinates = location['coordinates'] except: coordinates = '' latitude = 0 longitude = 0 address = '' if coordinates is not '': latitude = coordinates[0] longitude = coordinates[1] address = location['address'] + location['name'] location = model_im.Location() feed.location_id = location.location_id # 地址ID[INT] location.latitude = latitude location.longitude = longitude location.address = address self.db_insert_table_location(location) #发送者 senderid = self._db_record_get_int_value( rec, 'owner_primaryKeyID') if senderid == -1: senderid = -2 #是否给该条动态点赞 isliked = self._db_record_get_int_value(rec, 'haveLiked') #动态文本 content = self._db_record_get_string_value(rec, 'value') feed.account_id = userid # 账号ID[TEXT] feed.sender_id = senderid # 发布者ID[TEXT] feed.content = content # 文本[TEXT] if re.findall("image", media_url): feed.image_path = media_url elif re.findall("video", media_url): feed.video_path = media_url feed.send_time = createtime # 发布时间[INT] feed.likecount = likescount # 赞数量[INT] feed.commentcount = commentcount # 评论数量[INT] feed.comment_id = feedpk feed.deleted = rec.IsDeleted self.db_insert_table_feed(feed) except: pass self.db_commit() except: pass
def parse_Feed(self, plist_node=None): ''' 5A249183-668C-4CC0-B983-C0A7EA2E657F\ Library\Caches\PrivateStore\ P_u423af962f1456db6cba8465cf82bb91b\jp.naver.myhome.MBDataResults\ myhomelist timeline ''' ''' self.account_id = None # 账号ID[TEXT] self.sender_id = None # 发布者ID[TEXT] self.type = None # 动态类型[INT] self.content = None # 动态内容[TEXT] self.media_path = None # 媒体文件地址[TEXT] self.urls = None # 链接地址[json TEXT] json string ['url1', 'url2'...] # 预览地址[json TEXT] json string ['url1', 'url2'...] self.preview_urls = None self.attachment_title = None # 附件标题[TEXT] self.attachment_link = None # 附件链接[TEXT] self.attachment_desc = None # 附件描述[TEXT] self.send_time = None # 发布时间[INT] self.likes = None # 赞[TEXT] 逗号分隔like_id 例如:like_id, like_id, like_id, ... self.likecount = None # 赞数量[INT] self.rtcount = None # 转发数量[INT] # 评论[TEXT] 逗号分隔comment_id 例如:comment_id,comment_id,comment_id,... self.comments = None self.commentcount = None # 评论数量[INT] self.device = None # 设备名称[TEXT] self.location = None # 地址ID[TEXT] ''' def _print_plist(n, ind=''): print ind + 'key: {}'.format(n.Key) if n.Values: for chch in n.Values: _print_plist(chch, ind=ind + '|----') else: print ind + 'val:{}'.format(n.Value) bplist = BPReader.GetTree(plist_node.Data) if plist_node else None if not bplist: return for feed_node in bplist: #_print_plist(feed_node) feed = model_im.Feed() feed.account_id = self.cur_account_id # 账号ID[TEXT] feed.sender_id = feed_node['fromUser'].Value # 发布者ID[TEXT] feed.content = feed_node['contents']['text'].Value # 动态内容[TEXT] # 图片 视频 media_node = feed_node['contents']['media'] try: if media_node: for key in ('photos', 'videos'): # extra 'medias' if not media_node[key]: continue path_res = [] for each_media_node in media_node[key]: _path = each_media_node['sourceURL'].Value if _path: path_res.append(_path) if key == 'photos': feed.image_path = ','.join(x for x in path_res) elif key == 'videos': feed.video_path = ','.join(x for x in path_res) except: exc() # _print_plist(feed_node) # 链接 additional_node = feed_node['contents']['additionalContents'] if additional_node and additional_node['url']: feed.url = additional_node['url']['targetUrl'].Value if feed.url and additional_node['title']: feed.url_title = additional_node['title'].Value # location loc_node = feed_node['contents']['textLocation']['location'] if loc_node and loc_node['latitude'] and loc_node['longitude']: location = model_im.Location() location.address = loc_node['name'].Value location.timestamp = feed_node['postInfo']['createdTime'].Value location.latitude = loc_node['latitude'].Value location.longitude = loc_node['longitude'].Value try: self.csm.db_insert_table_location(location) except: exc() feed.location = location.location_id feed.send_time = feed_node['postInfo'][ 'createdTime'].Value # 发布时间[INT] feed.likecount = feed_node['postInfo'][ 'likeCount'].Value # 赞数量[INT] feed.rtcount = feed_node['postInfo']['sharedCount']['toPost'].Value \ + feed_node['postInfo']['sharedCount']['toTalk'].Value # 转发数量[INT] feed.commentcount = feed_node['postInfo'][ 'commentCount'].Value # 评论数量[INT] feed.source = plist_node.AbsolutePath try: self.csm.db_insert_table_feed(feed) except: exc() self.csm.db_commit()