def decode_recover_message(self): table_name = self.table_name["messages"] if not self.recovering_helper.is_valid(): return ts = self.recovering_helper.fetch_table(table_name, { "nsp_data": "Text", }) for rec in self.recovering_helper.read_deleted_record(ts): if canceller.IsCancellationRequested: return try: message_info = json.loads(rec["nsp_data"].Value) message = model_im.Message() message.source = self.checking_col.db_path message.msg_id = message_info.get("cuid", None) message.account_id = self.using_account.account_id message.talker_id = message_info.get("conversationId").split(":", 1)[1] message.sender_id = message_info.get("creator").split(":", 1)[1] message.sender_name = self.__query_sender_name(message_info.get("creator")) message.is_sender = 1 if message.account_id == message.sender_id else 0 message.content = self.__convert_message_content(message_info.get("content", None), message) message.send_time = self.__convert_timestamp(message_info.get("createdTime", None)) if not message.type: message.type = self.__convert_message_content_type(message_info.get("messagetype", None)) message.talker_type = model_im.CHAT_TYPE_GROUP if "@" in message.talker_id else model_im.CHAT_TYPE_FRIEND if message.type in (model_im.MESSAGE_CONTENT_TYPE_IMAGE, model_im.MESSAGE_CONTENT_TYPE_VIDEO, model_im.MESSAGE_CONTENT_TYPE_VOICE): self.__add_media_path(message) message.deleted = 1 self.model_im_col.db_insert_table_message(message) except Exception as e: pass self.model_im_col.db_commit()
def parse_chat_recovery(self, db_node, account_id, chat_dict, friend_dict, grp_dict): if db_node is None: return db = SQLiteParser.Database.FromNode(db_node) for c in chat_dict: is_grp = grp_dict.__contains__(c) print(chat_dict[c]) ts = SQLiteParser.TableSignature(chat_dict[c]) SQLiteParser.Tools.AddSignatureToTable(ts, "m_msginfo", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) #select m_msgid, m_msginfo, m_time, m_receive, m_type, m_remoteid from {} for rec in db.ReadTableDeletedRecords(ts, False): msg = model_im.Message() msg.msg_id = str(unity_c37r.try_get_rec_value(rec, 'm_msgid', '')) msg.content = str(unity_c37r.try_get_rec_value(rec, 'm_msginfo', '')) msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT is_rcv = int(unity_c37r.try_get_rec_value(rec, 'm_receive', 0)) msg.is_sender = 0 if is_rcv: msg.sender_id = account_id msg.is_sender = 1 elif not is_grp: msg.sender_id = c msg.sender_name = friend_dict[c].nickname else: msg.sender_id = int(unity_c37r.try_get_rec_value(rec, 'm_remoteid', 0)) if not is_grp: msg.talker_id = c msg.talker_name = friend_dict[c].nickname else: msg.talker_id = c msg.talker_name = grp_dict[c].name msg.send_time = int(unity_c37r.try_get_rec_value(rec, 'm_time', 0)) / 1000 msg.deleted = 1 self.im.db_insert_table_message(msg) self.im.db_commit()
def __recover_friend_messages(self): if not self.checking_account_recover_col.is_valid(): return ts = self.checking_account_recover_col.fetch_table("message_im_cc", { "MSG_ID": "Text", "MSG_TIME": "Int", "SENDER_ID": "Text", "CONV_CODE": "Text", "MSG_DATA": "Text", "MSG_TYPE": "Int", }) for rec in self.checking_account_recover_col.read_deleted_record(ts): try: m = model_im.Message() m.account_id = self.using_account.account_id m.msg_id = rec["MSG_ID"].Value m.send_time = Utils.convert_timestamp(rec["MSG_TIME"].Value) m.sender_id = rec["SENDER_ID"].Value m.sender_name = self.__fetch_sender_name(m.sender_id) m.source = self.checking_account_col.db_path m.talker_type = model_im.CHAT_TYPE_FRIEND m.type = self.__convert_message_type(rec["MSG_TYPE"].Value) m.talker_id = rec["CONV_CODE"].Value m.is_sender = 1 if m.sender_id in (self.using_account.account_id, self.using_account.username) else 0 m.content = self.__fetch_message_content(m, rec["MSG_TYPE"].Value, rec["MSG_DATA"].Value) m.deleted = 1 self.model_im_col.db_insert_table_message(m) except Exception: self.logger.error() self.model_im_col.db_commit()
def _generate_message_table(self): with self.checking_col as db_col: sql = """SELECT nsp_data FROM {};""".format(self.table_name["messages"]) db_col.execute_sql(sql) while db_col.has_rest(): try: message_info = json.loads(db_col.get_string(0)) message = model_im.Message() message.source = self.checking_col.db_path message.msg_id = message_info.get("cuid", None) message.account_id = self.using_account.account_id message.talker_id = message_info.get("conversationId").split(":", 1)[1] message.sender_id = message_info.get("creator").split(":", 1)[1] message.sender_name = self.__query_sender_name(message_info.get("creator")) message.is_sender = 1 if message.account_id == message.sender_id else 0 content = self.__convert_message_content(message_info.get("content", None), message) message.content = content message.send_time = self.__convert_timestamp(message_info.get("createdTime", None)) if not message.type: message.type = self.__convert_message_content_type(message_info.get("messagetype", None)) message.talker_type = model_im.CHAT_TYPE_GROUP if "@" in message.talker_id else model_im.CHAT_TYPE_FRIEND if message.type in (model_im.MESSAGE_CONTENT_TYPE_IMAGE, model_im.MESSAGE_CONTENT_TYPE_VIDEO, model_im.MESSAGE_CONTENT_TYPE_VOICE): self.__add_media_path(message) self.model_im_col.db_insert_table_message(message) except Exception as e: pass self.model_im_col.db_commit()
def _generate_message_table(self, node, account_id): sender_id = node source = node.AbsolutePath messages = self._open_json_file(node).get('messages', []) for m in messages: message = model_im.Message() message.source = source message.msg_id = m.get("clientmessageid", None) message.account_id = account_id message.talker_id = m.get("conversationid") message.sender_id = message.sender_name = sender_id = message.sender_name = sender_id = m.get( 'from').split('/')[-1] message.is_sender = 1 content = self.convert_message_content(m.get("content", None), message) message.content = content send_time = TimeHelper.str_to_ts(m.get('originalarrivaltime', None)[:-5].replace( 'T', " "), _format="%Y-%m-%d %H:%M:%S") message.send_time = send_time message.type = self.convert_message_content_type( m.get("messagetype", None)) message.talker_type = model_im.CHAT_TYPE_GROUP if "@" in message.talker_id else model_im.CHAT_TYPE_FRIEND # TODO media path 无法添加,案例数据没有 self.model_im_col.db_insert_table_message(message) self.model_im_col.db_commit()
def _add_unknown_resource(self): """ 因为无法找到本地文件缓存和message的对应方式,暂时以unknow的形式添加进去 :return: """ resource_path = self.root.PathWithMountPoint dir_name = os.path.dirname( os.path.dirname(os.path.dirname(resource_path))) resource_path = os.path.join(dir_name, "storage", "emulated", "0", "Telegram") file_dir_list = os.listdir(resource_path) for file_dir in file_dir_list: if file_dir == "Telegram Audio": file_type = model_im.MESSAGE_CONTENT_TYPE_VOICE elif file_dir == "Telegram Images": file_type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif file_dir == "Telegram Video": file_type = model_im.MESSAGE_CONTENT_TYPE_VIDEO else: file_type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT file_list = os.listdir(os.path.join(resource_path, file_dir)) file_list.remove(".nomedia") for _file in file_list: file_path = os.path.join(resource_path, file_dir, _file) message = model_im.Message() message.media_path = file_path message.type = file_type message.source = self.root.GetByPath( self.account_db_path).PathWithMountPoint self.model_col.db_insert_table_message(message) self.model_col.db_commit()
def _generate_message_table(self): chat_db = self._search_file_simple("chat-db$") if not chat_db: return Message.connect(chat_db) Friend.connect(chat_db) name = { member.user_id: member.nickname for member in Friend.objects.all } account_id = self.master_account.account_id for msg in Message.objects.all: try: message = model_im.Message() message.deleted = msg.deleted message.source = msg.source_path message.talker_id = msg.talker_id message.account_id = message.talker_id.split('##')[-1] message.content = msg.content message.msg_id = msg.msg_id message.sender_id = msg.sender_id message.send_time = TaoUtils.convert_timestamp(msg.ts) message.is_sender = 1 if int(account_id) == int( msg.sender_id) else 0 message.sender_name = name.get(int(message.sender_id), None) message.talker_type = model_im.CHAT_TYPE_SHOP message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE if msg.msg_type == 1 \ else model_im.MESSAGE_CONTENT_TYPE_TEXT message.media_path = msg.media_path self.model_im_col.db_insert_table_message(message) except Exception as e: self.logger.error() self.model_im_col.db_commit()
def decode_recover_people_messages(self): if not self.c_recover_helper.is_valid(): return ts = self.c_recover_helper.fetch_table('people_messages', { "msg_id": "Int", "conversation_uk": "Int", "msg_content": "Text", "msg_type": "Int", "uk": "Int", "uname": "Text", "ctime": "Int", "files_count": "Int", }) for rec in self.c_recover_helper.read_deleted_record(ts): if canceller.IsCancellationRequested: return try: message = model_im.Message() message.account_id = self.using_account.account_id message.deleted = 1 message.source = self.c_recover_helper.db_path message.sender_id = rec['uk'].Value message.sender_name = rec['uname'].Value message.talker_id = rec['conversation_uk'].Value message.msg_id = rec['msg_id'].Value message.content = rec['msg_content'].Value message.send_time = Utils.convert_timestamp(rec['ctime'].Value) message.is_sender = 1 if message.sender_id == self.using_account.uk else 0 message.talker_type = model_im.CHAT_TYPE_FRIEND self.model_im_col.db_insert_table_message(message) except Exception as e: pass self.model_im_col.db_commit()
def decode_recover_message(self): node = self.root.GetByPath("databases/__icssdk_database.db") if node is None: return db = SQLiteParser.Database.FromNode(node, canceller) if db is None: return table = '_MSG_' ts = SQLiteParser.TableSignature(table) for rec in db.ReadTableDeletedRecords(ts, False): if canceller.IsCancellationRequested: return try: message = model_im.Message() message.account_id = rec["localPin"].Value message.sender_id = rec["from_pin"].Value message.sender_name = rec["from_pin"].Value message.msg_id = rec["mid"].Value message.send_time = self.__convert_timestamp(rec["timestamp"].Value) message.source = self.user_db_path message_type = rec["body_type"].Value if message_type == "text": message.type = MESSAGE_CONTENT_TYPE_TEXT elif message_type == "image": message.content = rec["body_content"].Value self.__process_media(message) else: message.type = MESSAGE_CONTENT_TYPE_SYSTEM message.content = rec["body_content"].Value message.status = MESSAGE_STATUS_READ if rec["readed"].Value == 1 else MESSAGE_STATUS_UNREAD message.is_sender = 1 if message.account_id == message.sender_id else 0 self.model_im_col.db_insert_table_message(message) except Exception as e: print("error happen", e) self.model_im_col.db_commit()
def get_chats(self): ''' FieldName SQLType seqid Long PK msgid Varchar id Varchar fromid Varchar sessiontype Integer time Long status Integer direct Integer msgtype Integer content Varchar extra TEXT attachstr Varchar msgSvrId Long ''' dbPath = self.root.GetByPath(self.user_id + '/msg.db') if not self._read_db(node=dbPath): return for _id in self.friends.keys() or self.chatrooms.keys(): if 'msghistory' in self.cur_db.Tables: for rec in self._read_table('msghistory'): if (self._is_empty(rec, 'seqid', 'id') or self._is_duplicate(rec, 'seqid')): continue if _id != rec['id'].Value: continue friend = self.friends.get(_id) message = model_im.Message() message.deleted = 0 if rec.Deleted == DeletedState.Intact else 1 message.source = dbPath.AbsolutePath message.account_id = self.user_id message.talker_id = _id message.talker_type = model_im.CHAT_TYPE_FRIEND if _id in self.friends.keys( ) else model_im.CHAT_TYPE_GROUP message.talker_name = friend.nickname message.is_sender = model_im.MESSAGE_TYPE_SEND if rec[ 'fromid'].Value == self.user_id else model_im.MESSAGE_TYPE_RECEIVE message.sender_id = rec['fromid'].Value message.sender_name = self.username if message.is_sender == model_im.MESSAGE_TYPE_SEND else message.talker_name message.msg_id = rec['msgid'].Value message.type = self.parse_message_type( rec['msgtype'].Value) message.send_time = rec['time'].Value message.content = parse_yixin_msg_content( rec['content'].Value) message.media_path = self.get_media_path( rec['attachstr'].Value, message.type) if message.type == model_im.MESSAGE_CONTENT_TYPE_LOCATION: message.location_obj = message.create_location() message.location_id = self.get_location( message.location_obj, rec['content'].Value, rec['attachstr'].Value, message.send_time) self.csm.db_insert_table_message(message) self.csm.db_commit()
def create_empty_message(src_msg): msg = model_im.Message() msg.msg_id = src_msg.msg_id + random.randint(0, 0xffffffff) msg.send_time = src_msg.send_time msg.sender_id = src_msg.sender_id msg.is_sender = src_msg.is_sender msg.talker_id = src_msg.talker_id return msg
def _get_message_table(self): with self.agent_db_col as db_col: sql = """SELECT ZMRMESSAGE.ZHISTORYID, ZMRMESSAGE.ZTYPE, ZMRMESSAGE.ZTEXT, ZMRMESSAGE.ZTIME, ZMRMESSAGE.ZWASREAD, ZMRMESSAGE.ZPARTICIPANTUID, ZMRCONVERSATION.ZPID, ZMRMESSAGE.ZOUTGOING, ZMRMESSAGE.ZFILEID FROM ZMRMESSAGE LEFT JOIN ZMRCONVERSATION ON ZMRMESSAGE.ZCONVERSATION = ZMRCONVERSATION.Z_PK;""" db_col.execute_sql(sql) while db_col.has_rest(): try: message = model_im.Message() message.source = self.cl_db_path message.account_id, _, message.talker_id = db_col.get_string(6).split("|") message.msg_id = db_col.get_int64(0) message.content = db_col.get_string(2) message.send_time = self._get_timestamp(db_col.get_int64(3)) message.is_sender = 1 if db_col.get_int64(7) == 1 else 0 if "@" in message.talker_id: message.sender_id = db_col.get_string(5) message.talker_type = model_im.CHAT_TYPE_GROUP else: message.talker_type = model_im.CHAT_TYPE_FRIEND if message.is_sender == 1: message.sender_id = message.account_id else: message.sender_id = message.talker_id message.sender_name = self.__query_member_name(message.sender_id) _type = db_col.get_int64(1) if _type == 510: file_id = db_col.get_int64(8) file_info = self.__query_file_info(file_id) message.media_path = file_info[0] if not file_info[0] else file_info[2] if file_info[1] is not None: file_type = file_info[1].split("/")[0] if file_type == "image": message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif file_type == "audio": message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE elif file_type == "video": message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO else: message.type = None else: message.type = self.__convert_message_content_type(_type) self.model_im_col.db_insert_table_message(message) except Exception as e: print("debug error", e) self.model_im_col.db_commit()
def decode_recover_message(self): if not self.recover_helper.is_valid(): return ts = self.recover_helper.fetch_table( "MESSAGE_DATA", { "PROFILE_ID": "Text", "CONTACT_ID": "Text", "TYPE": "Int", "CONTENT": "Text", "TIMESTAMP": "Int", "SENDER": "Text", "HISTORY_ID": "Int", }) for rec in self.recover_helper.read_deleted_record(ts): if canceller.IsCancellationRequested: return try: message = model_im.Message() message.account_id = rec["PROFILE_ID"].Value message.talker_id = rec["CONTACT_ID"].Value message.sender_id = rec["SENDER"].Value message.msg_id = rec["HISTORY_ID"].Value message.send_time = self.__convert_timestamp( rec["TIMESTAMP"].Value) message.source = self.icq_data_path message.type = self.__convert_message_content_type( rec["TYPE"].Value) message.content = rec["CONTENT"].Value message.is_sender = 1 if message.account_id == message.sender_id else 0 message.deleted = 1 if message.type == model_im.MESSAGE_CONTENT_TYPE_VOIP: caller = message.sender_name if message.sender_name else message.sender_id responser = "您" if message.is_sender == 1: caller, responser = responser, caller call_info = json.loads(message.content) duration_time = call_info['duration'] videocall = call_info['videocall'] message.content = "{caller}呼叫了{responser} \n 持续运行了{s}秒".format( caller=caller, responser=responser, s=duration_time, ) self.model_im_col.db_insert_table_message(message) except Exception as e: print("error happen", e) self.model_im_col.db_commit()
def decode_recover_message(self): if not self.message_recover_helper.is_valid(): return ts = self.message_recover_helper.fetch_table("ZMRMESSAGE", { "ZHISTORYID": "Int", "ZTYPE": "Int", "ZTEXT": "Text", "ZTIME": "Int", "ZWASREAD": "Int", "ZPARTICIPANTUID": "Text", "ZOUTGOING": "Int", "ZFILEID": "Int", }) for rec in self.message_recover_helper.read_deleted_record(ts): if canceller.IsCancellationRequested: return try: message = model_im.Message() message.account_id = self.using_account.account_id message.source = self.cl_db_path message.msg_id = rec["ZHISTORYID"].Value message.content = rec["ZTEXT"].Value message.send_time = self._get_timestamp(rec["ZTIME"].Value) message.is_sender = 1 if rec["ZOUTGOING"].Value == 1 else 0 message.sender_id = message.account_id if message.is_sender == 1 else rec["ZPARTICIPANTUID"].Value message.sender_name = self.__query_member_name(message.sender_id) message.deleted = 1 _type = rec["ZTYPE"].Value if _type == 510: file_id = rec["ZFILEID"].Value file_info = self.__query_file_info(file_id) if file_info: message.media_path = file_info[0] if not file_info[0] else file_info[2] if file_info[1] is not None: file_type = file_info[1].split("/")[0] if file_type == "image": message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif file_type == "audio": message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE elif file_type == "video": message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO else: message.type = None else: message.type = self.__convert_message_content_type(_type) self.model_im_col.db_insert_table_message(message) except Exception as e: print("debug error", e) self.model_im_col.db_commit()
def __add_friend_messages(self): with self.checking_account_col as db_col: sql = """SELECT _id, MSG_ID, CLIENT_ID, MSG_TIME, MODIFY_TIME, SORTED_TIME, SENDER_ID, SENDER_TYPE, RECEIVER_ID, RECEIVER_TYPE, CONV_TARGET_TYPE, CONV_TARGET_ENTITY_TYPE, CONV_TARGET_CVS_TYPE, CONV_CODE, MSG_DATA, MSG_TYPE, READ_STATUS, SUMMARY, COLUMN_TYPE, LOCAL_DATA, EXT_INFO FROM message_im_cc;""" reader = db_col.execute_sql(sql) while reader.Read(): try: m = model_im.Message() m.account_id = self.using_account.account_id m.msg_id = db_col.get_string(1) m.send_time = Utils.convert_timestamp(db_col.get_int64(3)) m.sender_id = db_col.get_string(6) m.sender_name = self.__fetch_sender_name(m.sender_id) m.source = self.checking_account_col.db_path m.talker_type = model_im.CHAT_TYPE_FRIEND m.type = self.__convert_message_type(db_col.get_int64(15)) m.talker_id = db_col.get_string(13) m.is_sender = 1 if m.sender_id == self.using_account.account_id or m.sender_id.endswith( self.using_account.username) else 0 m.content = self.__fetch_message_content( m, db_col.get_int64(15), db_col.get_string(14)) self.model_im_col.db_insert_table_message(m) except Exception: self.logger.error() self.model_im_col.db_commit()
def _recover_message_record(self, account_id): recover_col = self.message_recover_col if not recover_col.is_valid(): return ts = recover_col.get_table( "message", { "msgid": [FieldType.Text, FieldConstraints.NotNull], "msgtime": [FieldType.Int, FieldConstraints.NotNull], "conversation": [FieldType.Text, FieldConstraints.NotNull], "isread": [FieldType.Int, FieldConstraints.NotNull], "status": [FieldType.Int, FieldConstraints.NotNull], "msgbody": [FieldType.Text, FieldConstraints.NotNull], "msgtype": [FieldType.Int, FieldConstraints.NotNull], "bodytype": [FieldType.Int, FieldConstraints.NotNull], "servertime": [FieldType.Int, FieldConstraints.NotNull], }) for rec in recover_col.read_deleted_records(ts): try: message = model_im.Message() message.account_id = account_id message.source = recover_col.db_path message.deleted = 1 message.msg_id = rec['msgid'].Value message.send_time = TaoUtils.convert_timestamp( rec['msgtime'].Value) message.status = model_im.MESSAGE_STATUS_READ if rec['isread'].Value == 1 \ else model_im.MESSAGE_STATUS_UNREAD message.talker_type = model_im.CHAT_TYPE_FRIEND message.talker_id = rec['conversation'].Value self._parse_message_body(message, rec['msgbody'].Value) message.sender_name = self._query_sender_name( message.account_id, message.sender_id) message.talker_name = message.sender_name if message.type == model_im.MESSAGE_CONTENT_TYPE_IMAGE: image_urls = message.media_path[:] for image_url in image_urls: message.media_path = image_url self.model_im_col.db_insert_table_message(message) else: self.model_im_col.db_insert_table_message(message) except Exception as e: self.logger.error() self.model_im_col.db_commit()
def __insert_people_messages(self): with self.cloud_p2p_col as db_col: sql = """SELECT conversation_uk, uk, name, avatar_url, msg_id, msg_content, msg_type, ctime, status, file_status, send_state, files_count, path, server_filename, server_mtime, thumbnail_url, server_ctime, category, image_prev_url2 FROM v_people_messages;""" db_col.execute_sql(sql) while db_col.has_rest(): try: message = model_im.Message() message.account_id = self.using_account.account_id message.source = db_col.db_path message.sender_id = db_col.get_int64(1) message.sender_name = db_col.get_string(2) message.talker_id = db_col.get_int64(0) message.msg_id = db_col.get_int64(4) message.content = db_col.get_string(5) message.send_time = Utils.convert_timestamp(db_col.get_int64(7)) message.is_sender = 1 if message.sender_id == self.using_account.uk else 0 message.talker_type = model_im.CHAT_TYPE_FRIEND if db_col.get_int64(11) > 0: self.__add_media_path(message, db_col.get_string(13)) self.model_im_col.db_insert_table_message(message) except Exception as e: pass self.model_im_col.db_commit()
def _generate_message_table(self): """生成消息的表""" db_col = self.message_col table = db_col.get_table( "message", { "msgid": [FieldType.Text, FieldConstraints.NotNull], "msgtime": [FieldType.Int, FieldConstraints.NotNull], "conversation": [FieldType.Text, FieldConstraints.NotNull], "isread": [FieldType.Int, FieldConstraints.NotNull], "status": [FieldType.Int, FieldConstraints.NotNull], "msgbody": [FieldType.Text, FieldConstraints.NotNull], "msgtype": [FieldType.Int, FieldConstraints.NotNull], "bodytype": [FieldType.Int, FieldConstraints.NotNull], "servertime": [FieldType.Int, FieldConstraints.NotNull], }) for rec in db_col.read_records(table): try: message = model_im.Message() message.account_id = os.path.basename( db_col.db_path).split(".")[0] message.source = db_col.db_path message.deleted = 1 if rec.IsDeleted else 0 message.msg_id = rec['msgid'].Value message.send_time = TaoUtils.convert_timestamp( rec['servertime'].Value) message.status = model_im.MESSAGE_STATUS_READ if rec['isread'].Value == 1 \ else model_im.MESSAGE_STATUS_UNREAD message.talker_type = model_im.CHAT_TYPE_SYSTEM if rec['conversation'].Value == "admin" \ else model_im.CHAT_TYPE_FRIEND message.talker_id = rec['conversation'].Value message.talker_name = message.sender_name self._parse_message_body(message, rec['msgbody'].Value) if rec['conversation'].Value != "admin": self._friend_array.add(message.talker_id) if message.type == model_im.MESSAGE_CONTENT_TYPE_IMAGE: image_urls = message.media_path[:] for image_url in image_urls: message.media_path = image_url self.model_im_col.db_insert_table_message(message) else: self.model_im_col.db_insert_table_message(message) except Exception as e: self.logger.error()
def _add_message_record(self, account_id): with self.message_col as db_col: sql = """SELECT msgid, msgtime, msgdirection, conversation, isread, isacked, isdelivered, islistened, status, msgbody, msgtype, bodytype, servertime FROM message;""" db_col.execute_sql(sql) while db_col.has_rest(): try: message = model_im.Message() message.account_id = account_id message.source = db_col.db_path message.msg_id = db_col.get_string(0) message.send_time = TaoUtils.convert_timestamp( db_col.get_int64(1)) message.status = model_im.MESSAGE_STATUS_READ if db_col.get_int64(4) == 1 \ else model_im.MESSAGE_STATUS_UNREAD message.talker_type = model_im.CHAT_TYPE_FRIEND message.talker_id = db_col.get_string(3) self._parse_message_body(message, db_col.get_string(9)) message.sender_name = message.talker_name = self._query_sender_name( message.account_id, message.sender_id) if message.type == model_im.MESSAGE_CONTENT_TYPE_IMAGE: image_urls = message.media_path[:] for image_url in image_urls: message.media_path = image_url self.model_im_col.db_insert_table_message(message) else: self.model_im_col.db_insert_table_message(message) except Exception as e: self.logger.error() self.model_im_col.db_commit()
def _get_message_table(self): with self.user_db_col as db_col: sql = """SELECT _id, UUID, localPin, type, datetime, timestamp, mid, from_pin, body_type, body_content, body_url, readed FROM _MSG_""" db_col.execute_sql(sql) while db_col.has_rest(): try: message = model_im.Message() message.account_id = db_col.get_string(2) message.sender_id = db_col.get_string(7) message.sender_name = db_col.get_string(7) message.msg_id = db_col.get_string(6) message.send_time = self.__convert_timestamp(db_col.get_int64(5)) message.source = self.user_db_path message_type = db_col.get_string(8) if message_type == "text": message.type = MESSAGE_CONTENT_TYPE_TEXT elif message_type == "image": message.content = db_col.get_string(10) self.__process_media(message) else: message.type = MESSAGE_CONTENT_TYPE_SYSTEM message.content = db_col.get_string(9) message.status = MESSAGE_STATUS_READ if db_col.get_int64(11) == 1 else MESSAGE_STATUS_UNREAD message.is_sender = 1 if message.account_id == message.sender_id else 0 self.model_im_col.db_insert_table_message(message) except Exception as e: print("debug error", e) self.model_im_col.db_commit()
def _generate_message_table(self): chat_db = self._search_file("Chat.sqlite$") if not chat_db: return for account_id in self.history_account_list: msg_model = create_message_table(account_id) member_model = create_chat_member_table(account_id) member_model.connect(chat_db) name = { member.user_id: member.nickname for member in member_model.objects.all } msg_model.connect(chat_db) for msg in msg_model.objects.all: try: message = model_im.Message() message.account_id = account_id message.deleted = msg.deleted message.talker_id = message.sender_id = msg.sender_id message.content = msg.content message.msg_id = msg.msg_id message.send_time = TaoUtils.convert_timestamp(msg.ts) message.is_sender = 1 if account_id == msg.sender_id else 0 message.sender_name = name.get(int(message.sender_id), None) message.talker_type = model_im.CHAT_TYPE_FRIEND message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE if msg.msg_type == 1 \ else model_im.MESSAGE_CONTENT_TYPE_TEXT message.media_path = self._search_file('{}$'.format( message.content)) self.model_im_col.db_insert_table_message(message) except Exception as e: self.logger.error() self.model_im_col.db_commit()
def parse(self, aid): files = self.cache + '/{}.db'.format(aid) if not os.path.exists(self.cache + '/{}.db'.format(aid)): self.m_print('no sql file,parse exit!') return conn = sql.SQLiteConnection( 'Data Source = {}; Readonly = True'.format(self.cache + '/{}.db'.format(aid))) conn.Open() cmd = sql.SQLiteCommand(conn) cmd.CommandText = ''' select iconMedia, nick, gender, city, mobile, real_name, dingtalkId, email, activeTime from tbuser where uid = {} '''.format(aid) reader = cmd.ExecuteReader() reader.Read() a = model_im.Account() a.account_id = aid a.nickname = GetString(reader, 1) if GetString(reader, 2) == 'F': a.gender = model_im.GENDER_FEMALE elif GetString(reader, 2) == 'M': a.gender = model_im.GENDER_MALE else: a.gender = model_im.GENDER_OTHER a.address = GetString(reader, 3) a.telephone = GetString(reader, 4) a.email = GetString(reader, 7) a.photo = GetString(reader, 0) self.im.db_insert_table_account(a) cmd.Dispose() cmd.CommandText = ''' select uid, iconMedia, nick, gender, city, mobile, email, extensation from tbuser where uid != {} '''.format(aid) reader = cmd.ExecuteReader() f_dict = dict() while reader.Read(): f = model_im.Friend() f.account_id = aid f.friend_id = GetInt64(reader, 0) f.photo = GetString(reader, 1) gender = GetString(reader, 3) if gender == 'F': f.gender = model_im.GENDER_FEMALE elif gender == 'M': f.gender = model_im.GENDER_MALE else: f.gender = model_im.GENDER_OTHER f.nickname = Getstring(reader, 2) f.address = GetString(reader, 4) f.telephone = GetString(reader, 5) f.email = GetString(reader, 6) f.source = files ext = Getstring(reaser, 7) js = json.loads(ext) if js.get('ownness') is not None: f.signature = js.get('ownness')[0].get('status') #self.im.db_insert_table_friend(f) f_dict[f.friend_id] = f cmd.Dispose() # 暂时未加入分组信息 cmd.CommandText = ''' select uid, empName, orgName, depName from tb_user_intimacy ''' reader = cmd.ExecuteReader() while reader.Read(): try: #f = model_im.Friend() f_dict[GetInt64(reader, 0)].remark = GetString(reader, 2) except: continue for k in f_dict: self.im.db_insert_table_friend(f_dict[k]) cmd.Dispose() cmd.CommandText = ''' select tbdingcontent.content, tbdingcontent.dingId, tbdinglist.senderId, tbdinglist.dingCreatedAt, tbdinglist.latestComments from tbdingcontent,tbdinglist where tbdingcontent.dingId = tbdinglist.dingId ''' reader = cmd.ExecuteReader() while reader.Read(): feed = model_im.Feed() feed.account_id = aid feed.content = GetString(reader, 0) feed.sender_id = GetInt64(reader, 2) feed.send_time = GetInt64(reader, 3) / 1000 feed.source = files string = GetString(reader, 4) if string is not '': js = json.loads(string) for a in js: feed.comments += '{},'.format(a.get('commentId')) fcm = model_im.FeedComment() fcm.comment_id = a.get('commentId') fcm.content = a.get('commentContent').get('text') fcm.sender_id = a.get('senderId') fcm.sender_name = a.get('commenter') fcm.create_time = a.get('createdAt') / 1000 fcm.source = files self.im.db_insert_table_feed_comment(fcm) self.im.db_insert_table_feed(feed) cmd.Dispose() conn.Close() self.im.db_commit() files = self.cache + '/{}_chat.db'.format(aid) if not os.path.exists(files): self.m_print('no chat sqlite file, parse exists!') return conn = sql.SQLiteConnection( 'Data Source = {}; ReadOnly = True'.format(files)) conn.Open() cmd = sql.SQLiteCommand(conn) cmd.CommandText(''' select cid, title , createAt, ownerId, groupIcon from tbconversation where type = 2 ''') reader = cmd.ExecuteReader() while reader.Read(): g = model_im.Chatroom() g.account_id = aid g.chatroom_id = GetString(reader, 0) g.name = GetString(reader, 1) g.owner_id = GetInt64(reader, 3) g.create_time = GetInt64(reader, 2) / 1000 g.source = files self.im.db_insert_table_chatroom(g) cmd.Dispose() cmd.CommandText = ''' select tbl_name from sqlite_master where type = 'table' and tbl_name like 'tbmsg_%' ''' reader = cmd.ExecuteReader() tbl_list = list() while reader.Read(): tbl_list.append(GetString(reade, 0)) cmd.Dispose() for tbl in tbl_list: cmd.CommandText = ''' select cid, mid, senderId, createdAt, contentType, content from {} '''.format(tbl) reader = cmd.ExecuteReader() while reader.Read(): msg = model_im.Message() msg.account_id = aid msg.msg_id = GetInt64(reader, 1) msg.talker_id = GetString(reader, 0) # talker_name? msg.sender_id = GetInt64(reader, 2) msg.is_sender = 1 if msg.sender_id == aid else 0 msg.send_time = GetInt64(reader, 3) / 1000 tp = GetInt64(reader, 4) string = GetString(reader, 5) if tp == 1: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT js = json.loads(string) msg.content = js.get('txt') elif tp == 2: msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE js = json.loads(string) msg.content = js.get('url') elif tp == 501 or tp == 503 or tp == 502: msg.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT msg.content = js.get('ext').get('f_name') elif tp == 901: # redpakcet #msg.type = model_im.message_content_type_ # pass js = json.loads(string) amount = js.get('ext').get('amount') title = js.get('ext').get('congrats') size = js.get('ext').get('size') elif tp == 203: # picture... msg.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT js = json.loads(string) msg.content = js.get('f_name') elif tp == 1200 or tp == 1203: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT js = json.loads(string) msg.content = js.get('markdown') elif tp == 300: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT js = json.loads(string) ext = js.get('multi')[0].get('ext') js = json.loads(ext) msg.content = js.get('b_tl') elif tp == 103: msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO js = json.loads(string) msg.content = js.get('url') # msg.media_path = msg.content # ??? elif tp == 1101: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT msg.content = string elif tp == 3: msg.type = model_im.MESSAGE_CONTENT_TYPE_VOICE msg.content = json.loads(string).get('url') elif tp == 1600: msg.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM msg.content = json.loads(string).get('dingContent') else: msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT msg.content = string self.im.db_insert_table_message(msg) self.im.db_commit() cmd.Dispose() conn.Close()
def __parse_message(self, a_node, aid): f_message_node = a_node.GetByPath('/{}_Message.sqlite'.format(aid)) db = unity_c37r.create_connection(f_message_node.PathWithMountPoint) cmd = sql.SQLiteCommand(db) cmd.CommandText = ''' select ZLOCAL_ID, ZBODY_TYPE, ZTIMESTAMP, ZBODY, ZEXT_ID, ZMSG_SENDER, ZMSG_TO, ZMSG_XML from ZMESSAGEV5OBJECT ''' try: reader = None reader = cmd.ExecuteReader() except: #因为此处会出现错误,故暂时在出异常时,把文件移到Cache里面(Md5) if reader is not None: reader.Close() cmd.Dispose() db.Close() dst_file = self.cache + "/" + unity_c37r.md5( f_message_node.PathWithMountPoint) unity_c37r.mapping_file_with_copy( f_message_node.PathWithMountPoint, dst_file) db = unity_c37r.create_connection(dst_file) cmd = sql.SQLiteCommand(db) cmd.CommandText = ''' select ZLOCAL_ID, ZBODY_TYPE, ZTIMESTAMP, ZBODY, ZEXT_ID, ZMSG_SENDER, ZMSG_TO, ZMSG_XML from ZMESSAGEV5OBJECT ''' reader = cmd.ExecuteReader() # 如果再出错误,这个案例就有问题了 while reader.Read(): if canceller.IsCancellationRequested: self.im.db_close() raise IOError("E") m = model_im.Message() m.talker_type = model_im.CHAT_TYPE_FRIEND m.account_id = aid target_id = unity_c37r.c_sharp_get_string(reader, 5) m.is_sender = 1 if str(target_id) == aid else 0 m.deleted = 0 m.msg_id = unity_c37r.c_sharp_get_long(reader, 0) m.send_time = int( unity_c37r.c_sharp_get_real(reader, 2) * 1000) / 1000 m.source = f_message_node.AbsolutePath m.sender_id = unity_c37r.c_sharp_get_string(reader, 5) if m.is_sender == 1: tid = unity_c37r.c_sharp_get_string(reader, 6) m.talker_id = tid.split('@')[0] else: m.talker_id = unity_c37r.c_sharp_get_string(reader, 5) #def __process_message(self, content, xml, aid, m_type, ext_id): p1 = unity_c37r.c_sharp_get_string(reader, 3) p2 = unity_c37r.c_sharp_get_string(reader, 7) tp = unity_c37r.c_sharp_get_long(reader, 1) p3 = unity_c37r.c_sharp_get_string(reader, 4) s_msg = self.__process_message(p1, p2, aid, tp, p3) m.content = s_msg.content m.media_path = s_msg.media_path m.type = s_msg.media_type try: self.im.db_insert_table_message(m) except: print('f****d!') cmd.Dispose() db.Close() self.im.db_commit() g_message_node = self.root_node.GetByPath( 'Documents/{}/account/{}_MUC.sqlite'.format(aid, aid)) if g_message_node is None: reader = None else: db = unity_c37r.create_connection( g_message_node.PathWithMountPoint) cmd = sql.SQLiteCommand(db) cmd.CommandText = ''' select ZBODY_TYPE, ZMSG_ID, ZSEND_TIME, ZBODY, ZSENDER_ID, ZMSG_TO, ZMSG_XML, ZEXT_ID from ZMUCMESSAGEOBJECT ''' reader = cmd.ExecuteReader() while reader is not None and reader.Read(): if canceller.IsCancellationRequested: self.im.db_close() raise IOError("E") m = model_im.Message() m.talker_type = model_im.CHAT_TYPE_GROUP m.msg_id = unity_c37r.c_sharp_get_string(reader, 1) m.account_id = aid sender_id = unity_c37r.c_sharp_get_string(reader, 4) m.sender_id = sender_id if sender_id is not 0 or unity_c37r.c_sharp_get_string( reader, 4) != '' else -1000 # -1000 means system... m.is_sender = 1 if str(m.sender_id) == aid else 0 m.send_time = unity_c37r.c_sharp_get_long(reader, 2) / 1000 m.source = g_message_node.AbsolutePath m.deleted = 0 tp = unity_c37r.c_sharp_get_long(reader, 0) xml = unity_c37r.c_sharp_get_string(reader, 6) ext_id = unity_c37r.c_sharp_get_string(reader, 7) if tp == 0 or tp == 3: m.content = unity_c37r.c_sharp_get_string(reader, 3) m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT if m.sender_id != -1000 else model_im.MESSAGE_CONTENT_TYPE_SYSTEM elif xml is not None and ext_id is None: if tp == 1: r = os.path.exists( os.path.join(self.root_node.PathWithMountPoint, 'Documents/image/%s' % ext_id)) if r: m.media_path = os.path.join( self.root_node.AbsolutePath, 'Documents/image/%s' % ext_id) m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif os.path.exists( os.path.join(self.root_node.PathWithMountPoint, 'Documents/image/th_%s' % ext_id)): m.media_path = os.path.join( self.root_node.AbsolutePath, 'Documents/image/th_%s' % ext_id) m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE else: m.content = 'image message not cached' m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT elif tp == 6: r = self.__handle_location(aid, ext_id) if r == '': m.content = 'location message not cached' m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT r = r.split(';') m.location_name = r[0] m.location_lat = r[1] m.location_lng = r[2] m.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION else: p1 = unity_c37r.c_sharp_get_string(reader, 3) p2 = unity_c37r.c_sharp_get_string(reader, 6) ext_id = unity_c37r.c_sharp_get_string(reader, 7) s_msg = self.__process_message(p1, p2, aid, tp, ext_id) m.media_path = s_msg.media_path m.type = s_msg.media_type self.im.db_insert_table_message(m) if reader is not None: cmd.Dispose() db.Close() self.im.db_commit()
def parse_Message(self, table_name, FRIEND_ID_NAME, CHATROOM_ID_NAME): ''' REAL_ + chat_id FieldName SQLType _key INTEGER _val BLOB ''' if table_name not in self.cur_db.Tables: return for rec in self._read_table(table_name): if self._is_empty(rec, '_key', '_val'): continue _val = rec['_val'].Value # tp(table_name + '_' + str(int(rec['_key'].Value))) msg_res = self._read_msg(_val, table_name) if not msg_res: return message = model_im.Message() message.account_id = self.cur_account_id message.talker_id = msg_res.get('talker_id', '') message.content = msg_res.get('content', None) message.sender_id = msg_res.get('sender_id', None) message.send_time = msg_res.get('send_time', None) message.sender_name = FRIEND_ID_NAME.get(message.sender_id, None) if msg_res.get('is_sender', None): message.is_sender = 1 message.status = model_im.MESSAGE_STATUS_SENT else: message.status = model_im.MESSAGE_STATUS_READ # talk_name, talk_type if message.talker_id.startswith('u'): message.talker_type = model_im.CHAT_TYPE_FRIEND message.talker_name = FRIEND_ID_NAME.get( message.talker_id, None) elif message.talker_id and message.talker_id[0] in ('c', 'r'): message.talker_type = model_im.CHAT_TYPE_GROUP message.talker_name = CHATROOM_ID_NAME.get( message.talker_id, None) else: continue # media if msg_res and 'media' in msg_res: message.media_path = msg_res.get('media', '') if 'image' in message.media_path: message.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE elif 'audio' in message.media_path: message.type = model_im.MESSAGE_CONTENT_TYPE_VOICE elif 'video' in message.media_path: message.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO elif msg_res.get('is_call', None): message.type = model_im.MESSAGE_CONTENT_TYPE_VOIP # attachment elif msg_res.get('attach_file_name', None): message.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT message.media_path = msg_res.get('attach_file_path', None) else: message.type = model_im.MESSAGE_CONTENT_TYPE_TEXT try: message.insert_db(self.im) except: exc() self.im.db_commit()
def get_chats(self): if self.user is None: return dbPath = self.root.GetByPath('/Documents/DB/' + self.user + '/chat.sqlite') db = SQLiteParser.Database.FromNode(dbPath) if db is None: return for id in self.friends.keys(): if 'r_s_chat_message_persistence_object' in db.Tables: ts = SQLiteParser.TableSignature( 'r_s_chat_message_persistence_object') SQLiteParser.Tools.AddSignatureToTable( ts, "msg_key", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) for rec in db.ReadTableRecords(ts, self.extract_deleted): if canceller.IsCancellationRequested: return if id != str(rec['from_user_id'].Value): if id != str(rec['to_user_id'].Value): continue friend = self.friends.get(id) message = model_im.Message() message.deleted = 0 if rec.Deleted == DeletedState.Intact else 1 message.source = dbPath.AbsolutePath message.account_id = self.user message.is_sender = model_im.MESSAGE_TYPE_SEND if rec[ 'from_user_id'].Value == self.user else model_im.MESSAGE_TYPE_RECEIVE message.talker_id = id if friend.type == model_im.FRIEND_TYPE_FRIEND: message.talker_type = model_im.CHAT_TYPE_FRIEND if friend.type == model_im.FRIEND_TYPE_SUBSCRIBE: message.talker_type = model_im.CHAT_TYPE_OFFICIAL message.talker_name = friend.nickname message.sender_id = rec['from_user_id'].Value message.sender_name = rec['fname'].Value message.type = self.parse_message_type( rec['class_type'].Value, rec['child_node_string'].Value) message.content = rec['summary'].Value message.send_time = int(float(rec['time_stamp'].Value)) message.media_path = self.get_media_path( id, message.is_sender, rec['child_node_string'].Value, rec['elements'].Value, message.type, message.send_time, message.deleted, message.repeated) message.msg_id = str(uuid.uuid1()).replace('-', '') self.im.db_insert_table_message(message) for id in self.chatrooms.keys(): if 'r_s_chat_message_persistence_object' in db.Tables: ts = SQLiteParser.TableSignature( 'r_s_chat_message_persistence_object') SQLiteParser.Tools.AddSignatureToTable( ts, "msg_key", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) for rec in db.ReadTableRecords(ts, self.extract_deleted): if canceller.IsCancellationRequested: return if id != str(rec['from_user_id'].Value): if id != str(rec['to_user_id'].Value): continue chatroom = self.chatrooms.get(id) message = model_im.Message() message.deleted = 0 if rec.Deleted == DeletedState.Intact else 1 message.source = dbPath.AbsolutePath message.account_id = self.user message.is_sender = model_im.MESSAGE_TYPE_SEND if rec[ 'from_user_id'].Value == self.user else model_im.MESSAGE_TYPE_RECEIVE message.talker_id = id message.talker_type = model_im.CHAT_TYPE_GROUP message.talker_name = chatroom.name message.sender_id = rec['from_user_id'].Value message.sender_name = rec['fname'].Value message.type = self.parse_message_type( rec['class_type'].Value, rec['child_node_string'].Value) message.content = rec['summary'].Value message.send_time = int(float(rec['time_stamp'].Value)) message.media_path = self.get_media_path( id, message.is_sender, rec['child_node_string'].Value, rec['elements'].Value, message.type, message.send_time, message.deleted, message.repeated) message.msg_id = str(uuid.uuid1()).replace('-', '') self.im.db_insert_table_message(message) self.im.db_commit()
def get_recover_groups_chat(self): gmessage_node = self.root.GetByPath("threads_db2") if gmessage_node is None: return db = SQLiteParser.Database.FromNode(gmessage_node, canceller) if db is None and "messages" not in db.Tables: return tbs = SQLiteParser.TableSignature("messages") SQLiteParser.Tools.AddSignatureToTable( tbs, "thread_key", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) SQLiteParser.Tools.AddSignatureToTable( tbs, "msg_type", SQLiteParser.FieldType.Int, SQLiteParser.FieldConstraints.NotNull) SQLiteParser.Tools.AddSignatureToTable( tbs, "msg_id", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) fs = self.root.FileSystem for rec in db.ReadTableDeletedRecords(tbs, False): if canceller.IsCancellationRequested: return send_info = rec["sender"].Value if "sender" in rec else None msg_type = rec["msg_type"].Value if "msg_type" in rec else None if "thread_key" in rec and rec["thread_key"].Value.find( "GROUP") != -1 and send_info: # 群聊天 message = model_im.Message() message.deleted = 1 message.account_id = self.account_id message.talker_type = 2 message.msg_id = rec[ "msg_id"].Value if "msg_id" in rec else None thread_key = rec["thread_key"].Value message.talker_id = thread_key if "sender" in rec and (not rec["sender"].IsDBNull): send_info = rec["sender"].Value if send_info: send_dict = json.loads(send_dict) if "name" in send_dict: message.sender_name = send_dict.get("name") if "user_key" in send_dict: message.sender_id = send_dict.get("user_key") if message.sender_id == self.account_id: message.is_sender = 1 if "text" in rec and (not recp["Text"].IsDBNull): message.content = rec["text"].Value message.type = 1 if "timestamp_sent_ms" in rec and ( not rec["timestamp_sent_ms"].IsDBNull): message.send_time = int( str(rec["timestamp_sent_ms"].Value)[:-3]) else: if "timestamp_ms" in rec and ( not rec["timestamp_ms"].IsDBNull): message.send_time = int( str(rec["timestamp_ms"].Value)[:-3]) if "pending_send_media_attachment" in rec and ( not rec["pending_send_media_attachment"].IsDBNull): attachment = rec["pending_send_media_attachment"].Value if attachment: attachment_str = json.loads(attachment) attachment_dict = attachment_str[0] if "type" in attachment_dict: tmp_type = attachment_dict["type"] if tmp_type == "VIDEO": message.type = 3 elif tmp_type == "PHOTO": message.type = 2 elif tmp_type == "VIDEO": message.type = 4 if "uri" in attachment_dict: media_path = attachment_dict.get("uri") if media_path.startswith("file:"): lists = re.split("/", media_path) media_name = lists[-1] media_path_list = fs.Search(media_name) for i in media_path_list: message.media_path = i.AbsolutePath elif media_path.startswith("http"): message.media_path = media_path if "shares" in rec and (not rec["shares"].IsDBNull): share_list = json.loads(rec["shares"].Value) share_dict = share_list[0] if "media" in share_dict: media_info = share_dict.get("media") if "type" in media_info[0] and media_info[0][ "type"] == "VIDEO": message.type = 4 if "type" in media_info[0] and media_info[0][ "type"] == "LINK": message.type = 8 if "href" in share_dict: message.media_path = share_dict["href"] if "name" in share_dict: message.content = share_dict.get("name") try: self.facebook_db.db_insert_table_message(message) except Exception as e: print(e) elif send_info == "" and msg_type == -1: # 无效的信息 continue self.facebook_db.db_commit()
def get_groups_chat(self): groups_chat = self.root.GetByPath("threads_db2") if groups_chat is None: return conn = System.Data.SQLite.SQLiteConnection( "Data Source = {0}; ReadOnly = True".format( groups_chat.PathWithMountPoint)) try: conn.Open() cmd = System.Data.SQLite.SQLiteCommand(conn) cmd.CommandText = ''' select messages.thread_key, text, sender, messages.timestamp_ms, messages.timestamp_sent_ms, shares, msg_type,messages.attachments from messages, folders where messages.thread_key = folders.thread_key and messages.thread_key like "GROUP%" ORDER BY messages.timestamp_ms ''' reader = cmd.ExecuteReader() fs = self.root.FileSystem while reader.Read(): try: if canceller.IsCancellationRequested: return message = model_im.Message() message.source = groups_chat.AbsolutePath message.account_id = self.account_id send_info = GetString(reader, 2) if GetString(reader, 2) else None msg_type = GetInt64(reader, 6) if GetInt64(reader, 6) else None if send_info: send_dic = json.loads(send_info) if "name" in send_dic: message.sender_name = send_dic.get("name") if "user_key" in send_dic: message.sender_id = send_dic.get("user_key") if send_dic.get("user_key") == self.account_id: message.is_sender = 1 message.talker_id = GetString(reader, 0) message.content = GetString(reader, 1) if GetString(reader, 1): message.type = 1 if GetInt64(reader, 4): message.send_time = int( str(GetInt64(reader, 4))[:-3]) else: message.send_time = int( str(GetInt64(reader, 3))[:-3]) message.talker_type = 2 # 群聊天 attachment = GetString(reader, 7) if attachment: attachment_str = json.loads(attachment) attachment_dict = attachment_str[0] if "type" in attachment_dict: tmp_type = attachment_dict["type"] if tmp_type == "VIDEO": message.type = 3 elif tmp_type == "PHOTO": message.type = 2 elif tmp_type == "VIDEO": message.type = 4 if "uri" in attachment_dict: media_path = attachment_dict.get("uri") if media_path.startswith("file:"): lists = re.split("/", media_path) media_name = lists[-1] media_path_list = fs.Search(media_name) if media_path_list: for i in media_path_list: message.media_path = i.AbsolutePath message.content = media_name else: message.content = media_name + "(本地资源已被删除)" elif media_path.startswith("http"): message.media_path = media_path elif "mime_type" in attachment_dict: if attachment_dict[ "mime_type"] == "image/jpeg": message.type = 2 elif attachment_dict[ "mime_type"] == "audio/mpeg": message.type = 3 if "filename" in attachment_dict: media_name = attachment_dict.get( "filename") media_path_list = fs.Search(media_name) if media_path_list: for i in media_path_list: message.media_path = i.AbsolutePath message.content = media_name else: message.content = media_name + "(本地资源已被删除)" if GetString(reader, 5): share_list = json.loads(GetString(reader, 5)) share_dict = share_list[0] if "media" in share_dict: media_info = share_dict.get("media") if "type" in media_info[0] and media_info[0][ "type"] == "VIDEO": message.type = 4 if "type" in media_info[0] and media_info[0][ "type"] == "LINK": message.type = 8 if "href" in share_dict: message.media_path = share_dict["href"] if "name" in share_dict: message.content = share_dict.get("name") try: self.facebook_db.db_insert_table_message(message) except Exception as e: print(e) #pass elif send_info == "" and msg_type == -1: # 无效的信息 continue except Exception as e: pass except Exception as e: pass #pass release_connection(reader, conn) self.facebook_db.db_commit()
def parse(self): if self.account is None: print('Get Account Id Failed, Parse Ret!') return False tg_node = self.root.GetByPath('Documents/tgdata.db') conn = sql.SQLiteConnection('Data Source = {}; Readonly=True'.format(tg_node.PathWithMountPoint)) conn.Open() cmd = sql.SQLiteCommand(conn) cmd.CommandText = ''' select uid, first_name, last_name, phone_number, photo_small, photo_medium, photo_big, last_seen, username from users_v29 where uid = {}'''.format(self.account) reader = cmd.ExecuteReader() if reader.Read(): a = model_im.Account() idx = 0 a.account_id = GetInt64(reader, 0) a.nickname = GetString(reader, 2) + GetString(reader, 1) a.telephone = GetString(reader, 3) p = self.get_photo_from_id(GetString(reader, 6)) if p is "": p = self.get_photo_from_id(GetString(reader, 5)) if p is "": p = self.get_photo_from_id(GetString(reader, 4)) a.photo = p a.username = GetString(reader, 8) self.im.db_insert_table_account(a) else: self.im.db_close() print("this is not the right group!") return False cmd.Dispose() cmd.CommandText = ''' select uid, first_name, last_name, phone_number, photo_small, photo_medium, photo_big, username from users_v29 where uid != {} '''.format(self.account) reader = cmd.ExecuteReader() while reader.Read(): f = model_im.Friend() f.account_id = self.account f.friend_id = GetInt64(reader, 0) f.nickname = GetString(reader, 2) + GetString(reader, 1) f.telephone = GetString(reader, 3) p = self.get_photo_from_id(GetString(reader, 6)) if p is "": p = self.get_photo_from_id(GetString(reader, 5)) if p is "": p = self.get_photo_from_id(GetString(reader, 4)) f.photo = p # user name? f.account_id = self.account self.im.db_insert_table_friend(f) cmd.Dispose() self.im.db_commit() # channels... cmd.CommandText = ''' select cid, data from channel_conversations_v29 ''' reader = cmd.ExecuteReader() while reader.Read(): g = model_im.Chatroom() g.account_id = self.account g.chatroom_id = GetInt64(reader, 0) blob = bytearray(GetBlob(reader, 1)) decoder = TeleBlobDecoder(blob) decoder.decode_long_for_ckey('i') # cid decoder.decode_long_for_ckey('ah') # acess hash decoder.decode_int_for_ckey('dv') decoder.decode_int_for_ckey('kind') decoder.decode_int_for_ckey('pts') decoder.decode_bytes_for_ckey('vsort', 9) decoder.decode_bytes_for_ckey('isort', 9) decoder.decode_bytes_for_ckey('usort', 9) decoder.decode_int_for_ckey('mread') decoder.decode_int_for_ckey('moutread') decoder.decode_int_for_ckey('mknown') decoder.decode_int_for_ckey('mlr') decoder.decode_int_for_ckey('mrd') decoder.decode_int_for_ckey('mrod') about = decoder.decode_string_for_ckey('about') g.description = about decoder.decode_string_for_ckey('username') decoder.decode_int_for_ckey('out') decoder.decode_int_for_ckey('unr') decoder.decode_int_for_ckey('der') decoder.decode_int_for_ckey('ds') decoder.decode_int_for_ckey('date') decoder.decode_int_for_ckey('from') decoder.decode_string_for_ckey('text') decoder.decode_data_for_ckey('media') # note that... decoder.decode_int_for_ckey('ucount') decoder.decode_int_for_ckey('sucount') g.name = decoder.decode_string_for_ckey('ct') # nick name ps = self.get_photo_from_id(decoder.decode_string_for_ckey('cp.s')) # photo small pm = self.get_photo_from_id(decoder.decode_string_for_ckey('cp.m')) # photo medium pb = self.get_photo_from_id(decoder.decode_string_for_ckey('cp.l')) # photo big if pb is not "": g.photo = pb elif pm is not "": g.photo = pm else: g.photo = ps decoder.decode_int_for_ckey('ver') decoder.decode_int_for_ckey('adm') decoder.decode_int_for_ckey('role') decoder.decode_int_for_ckey('ro') decoder.decode_long_for_ckey('flags') decoder.decode_int_for_ckey('lef') decoder.decode_int_for_ckey('kk') decoder.decode_int_for_ckey('mtci') decoder.decode_long_for_ckey('mtch') decoder.decode_string_for_ckey('rr') decoder.decode_int_for_ckey('pmi') decoder.decode_int_for_ckey('ccd') decoder.decode_int_for_ckey('pdt') self.im.db_insert_table_chatroom(g) cmd.Dispose() # Friend messages # 稳的一笔 C37R cmd.CommandText = ''' select mid, cid, message, media, from_id, to_id, outgoing, date from messages_v29 ''' reader = cmd.ExecuteReader() while reader.Read(): m = model_im.Message() m.talker_type = model_im.CHAT_TYPE_FRIEND m.account_id = self.account m.msg_id = GetInt64(reader, 0) m.is_sender = GetInt64(reader, 6) m.content = GetString(reader, 2) m.sender_id = GetInt64(reader, 4) m.talker_id = GetInt64(reader, 5) if m.is_sender == 1 else m.sender_id m.send_time = GetInt64(reader, 7) # parse media.... excited. try: blob = bytearray(GetBlob(reader, 3)) if blob is None or len(blob) == 0: pass else: self.parse_media_attachments(blob, m) except Exception as e: print(e) if m.content == '' or m.content is None: continue self.im.db_insert_table_message(m) self.im.db_commit() cmd.Dispose() cmd.CommandText = ''' select cid, data, mid from channel_messages_v29 ''' reader = cmd.ExecuteReader() while reader.Read(): m = model_im.Message() m.talker_type = model_im.CHAT_TYPE_GROUP m.account_id = self.account m.talker_id = GetInt64(reader, 0) bts = GetBlob(reader, 1) m.msg_id = GetInt64(reader, 2) decoder = TeleBlobDecoder(bts) decoder.decode_int_for_ckey('i') decoder.decode_bytes_for_ckey('sk', 17) decoder.decode_int_for_ckey('pts') decoder.decode_int_for_ckey('unr') decoder.decode_int_for_ckey('out') decoder.decode_int_for_ckey('ds') m.sender_id = decoder.decode_long_for_ckey('fi') to_id = decoder.decode_long_for_ckey('ti') cid = decoder.decode_long_for_ckey('ci') m.content = decoder.decode_string_for_ckey('t') m.send_time = decoder.decode_int_for_ckey('d') medias = decoder.decode_data_for_ckey('md') try: if len(medias) == 0 or medias is None: pass else: self.parse_media_attachments(medias, m) except Exception as e: print(e) decoder.decode_int_for_ckey('rd') decoder.decode_long_for_ckey('ri') decoder.decode_int_for_ckey('lt') decoder.decode_long_for_ckey('f') decoder.decode_int_for_ckey('sqi') decoder.decode_int_for_ckey('sqo') if m.content is '' or m.content is None: continue self.im.db_insert_table_message(m) self.im.db_commit() return True
def get_group_messages(self): if self.read_file_path(): conn = System.Data.SQLite.SQLiteConnection( "Data Source = {0}; ReadOnly = True".format( self.source_path.PathWithMountPoint)) else: conn = System.Data.SQLite.SQLiteConnection( "Data Source = {0}".format(self.dest_path)) try: conn.Open() cmd = System.Data.SQLite.SQLiteCommand(conn) cmd.CommandText = """ select key_remote_jid as group_id, key_from_me, data, remote_resource, timestamp, media_url, media_mime_type, media_wa_type, media_name ,latitude ,longitude, status from messages where key_remote_jid in (select distinct(gjid) from group_participants a where jid is not null) """ reader = cmd.ExecuteReader() fs = self.root.FileSystem while reader.Read(): if canceller.IsCancellationRequested: return if GetString(reader, 0).find("broadcast") != -1: continue message = model_im.Message() message.source = self.source_path.AbsolutePath message.talker_type = 2 # 群聊天 message.account_id = self.account_id groups_id = GetString(reader, 0) message.is_sender = GetInt64(reader, 1) media_type = GetString(reader, 7) content = GetString(reader, 2) message.media_path = "" send_time = int(str(GetInt64(reader, 4))[:-3]) message.content = content message.talker_id = GetString(reader, 0) status = GetInt64(reader, 12) if groups_id in self.groups_dicts: message.talker_name = self.groups_dicts[groups_id] if GetInt64(reader, 1) == 1 and GetString(reader, 3) == "": message.sender_id = self.account_id message.sender_name = self.account_pushname elif GetInt64(reader, 1) == 1 and GetString(reader, 3) != "": message.sender_id = GetString(reader, 3) if GetString(reader, 3) in self.contacts_dicts: message.sender_name = self.contacts_dicts[GetString( reader, 3)] else: message.sender_name = GetString(reader, 3) elif GetInt64(reader, 1) == 0: sender_id = GetString(reader, 3) message.sender_id = sender_id if sender_id in self.contacts_dicts: message.sender_name = self.contacts_dicts[sender_id] else: message.sender_name = GetString(reader, 3) if status == 1: message.status = 1 else: message.status = 0 if media_type == '0': message.type = 1 elif media_type == '1': message.type = 2 if GetString(reader, 8): img_name = GetString(reader, 8) img_node = fs.Search(img_name) for i in img_node: img_path = i.AbsolutePath message.media_path = img_path elif media_type == '2': message.type = 3 if GetString(reader, 8): voice_name = GetString(reader, 8) voice_node = fs.Search(voice_name) for i in voice_node: voice_path = i.AbsolutePath message.media_path = voice_path elif media_type == '3': message.type = 4 if GetString(reader, 8): video_name = GetString(reader, 8) video_node = fs.Search(video_name) for i in video_node: video_path = i.AbsolutePath message.media_path = video_path elif media_type == '5': if GetFloat(reader, 9) > 0 and GetFloat(reader, 10) > 0: location = model_im.Location() message.type = 7 message.location_id = location.location_id lat, lng = unity_c37r.gcj2wgs_exact( GetFloat(reader, 9), GetFloat(reader, 10)) if lng: location.longitude = lng if lat: location.latitude = lat try: self.whatsapp.db_insert_table_location(location) except Exception as e: pass message.content = content message.send_time = send_time try: self.whatsapp.db_insert_table_message(message) except Exception as e: pass except Exception as e: TraceService.Trace(TraceLevel.Error, "{0}".format(e)) if conn != None: conn.Close() self.whatsapp.db_commit()
def get_chats(self): if self.user is None: return dbPath = self.root.GetByPath('../../../Documents/' + self.user + '/IMDataNew.Sqlite3') db = SQLiteParser.Database.FromNode(dbPath) if db is None: return if 'NewIMMessageInfo' in db.Tables: for contact_id in self.contacts.keys(): ts = SQLiteParser.TableSignature('NewIMMessageInfo') SQLiteParser.Tools.AddSignatureToTable(ts, "[uid]", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) for rec in db.ReadTableRecords(ts, self.extract_deleted, True): if canceller.IsCancellationRequested: return uid = self._db_record_get_string_value(rec, '[uid]') if str(contact_id) != uid: continue contact = self.contacts.get(contact_id) message = model_im.Message() message.deleted = 0 if rec.Deleted == DeletedState.Intact else 1 message.source = dbPath.AbsolutePath message.account_id = self.user message.msg_id = str(uuid.uuid1()).replace('-', '') message.talker_id = contact_id message.talker_name = contact.nickname message.talker_type = model_im.CHAT_TYPE_FRIEND msgtype = self._db_record_get_string_value(rec, '[msgtype]') message.is_sender = model_im.MESSAGE_TYPE_SEND if msgtype else model_im.MESSAGE_TYPE_RECEIVE message.sender_id = self.user if message.is_sender == model_im.MESSAGE_TYPE_SEND else contact_id message.sender_name = contact.nickname if message.sender_id == contact_id else self.username message.type = self.parse_message_type(self._db_record_get_int_value(rec, '[msgcontype]')) message.content = self.decode_url_message(self._db_record_get_string_value(rec, '[msgcontent]')) try: message.send_time = int(time.mktime(time.strptime(rec['[msgtime]'].Value, '%Y-%m-%d %H:%M:%S'))) except: pass message.media_path = self.get_media_path(message.type, message.content, contact_id) if message.type == model_im.MESSAGE_CONTENT_TYPE_LOCATION: message.location_obj = message.create_location() message.location_id = self.get_location(message.location_obj, message.content, message.send_time) self.im.db_insert_table_message(message) dbPath = self.root.GetByPath('../../../Documents/' + self.user + '/callHistoryRecord.Sqlite3') db = SQLiteParser.Database.FromNode(dbPath) if db is None: return if 'call' in db.Tables: for contact_id in self.contacts.keys(): ts = SQLiteParser.TableSignature('call') SQLiteParser.Tools.AddSignatureToTable(ts, "[uid]", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull) for rec in db.ReadTableRecords(ts, self.extract_deleted, True): if canceller.IsCancellationRequested: return uid = self._db_record_get_string_value(rec, '[uid]') if contact_id != uid: continue contact = self.contacts.get(contact_id) message = model_im.Message() message.deleted = 0 if rec.Deleted == DeletedState.Intact else 1 message.source = dbPath.AbsolutePath message.account_id = self.user message.msg_id = str(uuid.uuid1()).replace('-', '') message.talker_id = contact_id message.talker_name = contact.nickname message.talker_type = model_im.CHAT_TYPE_FRIEND calltype = self._db_record_get_int_value(rec, '[calltype]') message.is_sender = model_im.MESSAGE_TYPE_SEND if calltype == 0 else model_im.MESSAGE_TYPE_RECEIVE message.sender_id = self.user if message.is_sender == model_im.MESSAGE_TYPE_SEND else contact_id message.sender_name = contact.nickname if message.sender_id == contact_id else self.username message.type = model_im.MESSAGE_CONTENT_TYPE_VOIP if IsDBNull(rec['[telephone]'].Value): calltime = self._db_record_get_string_value(rec, '[calltime]') if(calltime == '00:00'): message.content = "[视频]已取消" else: message.content = "[视频]通话时长" + calltime else: calltime = self._db_record_get_string_value(rec, '[calltime]') if(calltime == '00:00'): message.content = "[语音]已取消" else: message.content = "[语音]通话时长" + calltime self.im.db_insert_table_message(message) self.im.db_commit()