def parse_feed(self, account_id):
db_node = self.node.GetByPath('feed60_{}'.format(account_id))
if db_node is None:
return
conn = unity_c37r.create_connection_tentatively(db_node, True)
cmd = SQLiteCommand(conn)
# 以下列类型全部为numeric类型，但是插入的各种类型都有
# fid, time, content, location(name), sender id
# sqlite 在数据管理方面是动态类型的，使用了更普遍的动态类型系统，sqlite中，值的类型和值本身是有关系的，与容器无关
# 大型数据库则使用更为刚性和静态的类型语言
cmd.CommandText = '''
select _id, field3, field55, field59, field10 from commonfeed
'''
reader = cmd.ExecuteReader()
while reader.Read():
feed = model_im.Feed()
feed.account_id = account_id
feed.deleted = 0
feed.source = db_node.AbsolutePath
content = json.loads(unity_c37r.c_sharp_get_string(reader, 2))
feed.content = content.get('text')
feed.send_time = unity_c37r.c_sharp_try_get_time(reader, 1) / 1000
feed.sender_id = unity_c37r.c_sharp_get_long(reader, 4)
feed.deleted = 0
feed.insert_db(self.im)
self.im.db_commit()
reader.Close()
cmd.Dispose()
conn.Close()
#recovery
db = SQLiteParser.Database.FromNode(db_node)
ts = SQLiteParser.TableSignature('commonfeed')
SQLiteParser.Tools.AddSignatureToTable(ts, "field55", SQLiteParser.FieldType.Text, SQLiteParser.FieldConstraints.NotNull)
for rec in db.ReadTableDeletedRecords(ts, False):
feed = model_im.Feed()
feed.account_id = account_id
feed.deleted = 1
content = unity_c37r.try_get_rec_value(rec, "field55", "")
try:
js = json.loads(content)
feed.content = js.get('text')
except:
feed.content = content
feed.send_time = unity_c37r.try_get_rec_value(rec, "field3", 0) / 1000
feed.sender_id = unity_c37r.try_get_rec_value(rec, "field10", 0)
feed.insert_db(self.im)
self.im.db_commit()
def parse_feed(self):
'''解析动态数据'''
feedNode = self.node.Parent.GetByPath('/ExploreFeed.db')
try:
db = SQLiteParser.Database.FromNode(feedNode, canceller)
if db is None:
return
ts = SQLiteParser.TableSignature('XYExploreNote')
self.parse_feed_image()
self.parse_feed_video()
for rec in db.ReadTableRecords(ts, self.extractDeleted, True):
try:
feed = model_im.Feed()
feed.account_id = self.account_id
sender_id = re.sub(
'XYPHUser&', '',
self._db_record_get_string_value(rec, 'user'))
feed.sender_id = self.id2userid[sender_id]
feed.content = self._db_record_get_string_value(
rec, 'desc')
image_pathes = self._db_record_get_string_value(
rec, 'imagesList')
video_pathes = self._db_record_get_string_value(
rec, 'videoInfo')
image_urls = []
video_urls = []
for image_path in image_pathes.split(','):
try:
image_path = re.sub('6#XYPHNoteImageInfo&', '',
image_path)
image_urls.append(self.image_url[image_path])
except:
pass
feed.image_path = ','.join(image_urls)
for video_path in video_pathes.split(','):
try:
video_path = re.sub('XYPHVideoModel&', '',
video_path)
video_urls.append(self.video_url[video_path])
except:
pass
feed.video_path = ','.join(video_urls)
feed.url_desc = self._db_record_get_string_value(
rec, 'displayTitle')
feed.send_time = self._get_timestamp(
self._db_record_get_string_value(rec, 'cursor_score'))
feed.likecount = self._db_record_get_int_value(
rec, 'likes')
feed.deleted = rec.IsDeleted
self.db_insert_table_feed(feed)
except:
pass
self.db_commit()
except Exception as e:
print(e)
def parse_recovery(self, aid):
db_node = self.node.GetByPath('databases/{}'.format(
self.account_dbs[aid]))
if db_node is None:
return
sp = SQLiteParser.Database.FromNode(db_node)
ts = SQLiteParser.TableSignature('users')
SQLiteParser.Tools.AddSignatureToTable(
ts, "user_id", SQLiteParser.FieldType.Int,
SQLiteParser.FieldConstraints.NotNull)
for rec in sp.ReadTableDeletedRecords(ts, False):
f = model_im.Friend()
f.account_id = aid
f.friend_id = unity_c37r.try_get_rec_value(rec, 'user_id', 0)
f.nickname = unity_c37r.try_get_rec_value(rec, 'name', '')
f.remark = unity_c37r.try_get_rec_value(rec, 'username', '')
f.signature = unity_c37r.try_get_rec_value(rec, 'description', '')
f.photo = unity_c37r.try_get_rec_value(rec, 'image_url')
f.deleted = 1
self.im.db_insert_table_friend(f)
self.im.db_commit()
ts = SQLiteParser.TableSignature('statuses')
SQLiteParser.Tools.AddSignatureToTable(
ts, "content", SQLiteParser.FieldType.Blob,
SQLiteParser.FieldConstraints.NotNull)
for rec in sp.ReadTableDeletedRecords(ts, False):
feed = model_im.Feed()
feed.account_id = aid
feed.deleted = 1
try:
feed.content = unity_c37r.try_get_rec_value(rec, 'content', '')
feed.content = feed.content.decode('utf-8', 'ignore')
feed.sender_id = unity_c37r.try_get_rec_value(
rec, 'author_id', 0)
feed.send_time = unity_c37r.try_get_rec_value(
rec, 'created', 0) / 1000
feed.likecount = unity_c37r.try_get_rec_value(
rec, 'favorite_count', 0)
feed.rtcount = unity_c37r.try_get_rec_value(
rec, 'retweet_count', 0)
self.im.db_insert_table_feed(feed)
except:
traceback.print_exc()
self.im.db_commit()
def parse_recovery(self):
for aid in self.account:
db_path = self.db_dict[aid]
node = self.node.GetByPath(db_path)
if node is None:
continue
db = SQLiteParser.Database.FromNode(node)
ts = SQLiteParser.TableSignature('Users')
SQLiteParser.Tools.AddSignatureToTable(
ts, "id", SQLiteParser.FieldType.Int,
SQLiteParser.FieldConstraints.NotNull)
for rec in db.ReadTableDeletedRecords(ts, False):
f = model_im.Friend()
f.account_id = aid
f.friend_id = int(unity_c37r.try_get_rec_value(rec, "id", 0))
f.nickname = str(unity_c37r.try_get_rec_value(rec, "name", ""))
f.remark = str(
unity_c37r.try_get_rec_value(rec, "screenName", ""))
f.address = str(
unity_c37r.try_get_rec_value(rec, "location", ""))
f.signature = str(
unity_c37r.try_get_rec_value(rec, "description", ""))
f.deleted = 1
self.im.db_insert_table_friend(f)
self.im.db_commit()
ts = SQLiteParser.TableSignature('Statuses')
SQLiteParser.Tools.AddSignatureToTable(
ts, "userId", SQLiteParser.FieldType.Int,
SQLiteParser.FieldConstraints.NotNull)
for rec in db.ReadTableDeletedRecords(ts, False):
feed = model_im.Feed()
feed.account_id = aid
feed.sender_id = int(
unity_c37r.try_get_rec_value(rec, "userId", 0))
feed.content = str(
unity_c37r.try_get_rec_value(rec, "text", ""))
feed.send_time = int(
unity_c37r.try_get_rec_value(rec, "date", 0))
feed.deleted = 1
self.im.db_insert_table_feed(feed)
self.im.db_commit()
def parse_feed(self, dbPath):
'''解析动态数据'''
db = SQLite.SQLiteConnection(
'Data Source = {}; ReadOnly = True'.format(dbPath))
db.Open()
db_cmd = SQLite.SQLiteCommand(db)
fs = self.node.FileSystem
try:
if self.db is None:
return
db_cmd.CommandText = '''select address, userid, sendTime, nickname, icon, myuid, likecount, datetime, reviewcount,
gender, dynamicid, sendPhotos, content, deleted from DBTableName_MySendDynamic'''
sr = db_cmd.ExecuteReader()
while (sr.Read()):
try:
feed = model_im.Feed()
if canceller.IsCancellationRequested:
break
feed.account_id = self.account_id
feed.sender_id = self._db_reader_get_int_value(sr, 1)
feed.send_time = self._get_timestamp(
self._db_reader_get_int_value(sr, 2))
feed.deleted = self._db_reader_get_int_value(sr, 13)
feed.likecount = self._db_reader_get_int_value(sr, 6)
feed.content = self._db_reader_get_string_value(sr, 12)
feed.commentcount = self._db_reader_get_int_value(sr, 8)
if not IsDBNull(sr[11]):
mediapathes = Encoding.UTF8.GetString(sr[11])
feed.image_path = mediapathes
self.db_insert_table_feed(feed)
except:
traceback.print_exc()
sr.Close()
self.db_commit()
db_cmd.Dispose()
db.Close()
except Exception as e:
print(e)
def get_feeds(self):
if self.read_file_path():
conn = System.Data.SQLite.SQLiteConnection(
"Data Source = {0}; ReadOnly = True".format(
self.source_path.PathWithMountPoint))
else:
conn = System.Data.SQLite.SQLiteConnection(
"Data Source = {0}".format(self.dest_path))
try:
conn.Open()
cmd = System.Data.SQLite.SQLiteCommand(conn)
cmd.CommandText = """
select key_remote_jid as group_id, key_from_me, data, remote_resource, timestamp, media_url, media_caption, media_mime_type, media_wa_type, media_name ,latitude ,longitude, status from messages
"""
reader = cmd.ExecuteReader()
fs = self.root.FileSystem
while reader.Read():
if canceller.IsCancellationRequested:
return
try:
media_type = GetString(reader, 8)
feed_id = GetString(reader, 0)
# 好友状态
if feed_id.find("broadcast") == -1:
continue
feed = model_im.Feed()
feed.source = self.source_path.AbsolutePath
feed.account_id = self.account_id
if GetInt64(reader, 1) == 1:
feed.sender_id = self.account_id
else:
feed.sender_id = GetString(reader, 3)
if GetString(reader, 2):
feed.content = GetString(reader, 2)
if GetString(reader, 5):
feed.urls = json.dumps(GetString(reader, 5))
if GetString(reader, 6):
feed.content = GetString(reader, 6)
if media_type == '0':
feed.type = 1
elif media_type == '1':
feed.type = 2
if GetString(reader, 9):
img_name = GetString(reader, 9)
img_node = fs.Search(img_name)
for i in img_node:
img_path = i.AbsolutePath
feed.media_path = img_path
elif media_type == '2':
feed.type = 3
if GetString(reader, 9):
voice_name = GetString(reader, 9)
voice_node = fs.Search(voice_name)
for i in voice_node:
voice_path = i.AbsolutePath
feed.media_path = voice_path
elif media_type == '3':
feed.type = 4
if GetString(reader, 9):
video_name = GetString(reader, 9)
video_node = fs.Search(video_name)
for i in video_node:
video_path = i.AbsolutePath
feed.media_path = video_path
feed.send_time = int(str(GetInt64(reader, 4))[:-3])
except Exception as e:
TraceService.Trace(TraceLevel.Error, "{0}".format(e))
try:
self.whatsapp.db_insert_table_feed(feed)
except Exception as e:
pass
except Exception as e:
TraceService.Trace(TraceLevel.Error, "{0}".format(e))
if conn != None:
conn.Close()
self.whatsapp.db_commit()
def get_messages(self, node, account_id):
db = SQLiteParser.Database.FromNode(node, canceller)
if db is None:
return
if 'messageTable' not in db.Tables:
return
tbs = SQLiteParser.TableSignature("messageTable")
for rec in db.ReadTableRecords(tbs, self.extractDeleted, True):
try:
if canceller.IsCancellationRequested:
return
messages = model_im.Message()
messages.account_id = account_id
messages.source = node.AbsolutePath
if rec.Deleted == DeletedState.Deleted:
messages.deleted = 1
if "sessionId" in rec and (not rec["sessionId"].IsDBNull):
messages.talker_id = rec["sessionId"].Value
if rec["sessionId"].Value in self.friend_list.keys():
messages.talker_type = 1
messages.talker_name = self.friend_list[
rec["sessionId"].Value]
elif rec["sessionId"].Value in self.group_list.keys():
messages.talker_type = 2
messages.talker_name = self.group_list[
rec["sessionId"].Value]
if "fromId" in rec and (not rec["fromId"].IsDBNull):
messages.sender_id = rec["fromId"].Value
if rec["fromId"].Value == int(account_id):
messages.is_sender = 1
if "fromUserName" in rec and (
not rec["fromUserName"].IsDBNull):
messages.sender_name = rec["fromUserName"].Value
if "sendTime" in rec and (not rec["sendTime"].IsDBNull):
messages.send_time = convert_to_unixtime(
rec["sendTime"].Value)
if "messageType" in rec and (not rec["messageType"].IsDBNull):
messages.type = 1
msg_type = rec["messageType"].Value
if msg_type == 1: # text
messages.type = 1
messages.content = rec["messageContent"].Value
elif msg_type == 2: # img
messages.type = 2
if rec["messageContent"].Value.find("||") != -1:
messages.media_path = rec[
"messageContent"].Value.split("||")[1]
elif rec["messageContent"].Value == "deleted":
messages.content = rec["messageContent"].Value
else:
messages.media_path = rec["messageContent"].Value
elif msg_type == 3: # voice
messages.type = 3
if rec["messageContent"].Value.find("||") != -1:
messages.media_path = rec[
"messageContent"].Value.split("||")[1]
else:
messages.content = rec["messageContent"].Value
elif msg_type == 4: # location
messages.type = 7
try:
lng, lat, addr = rec["messageContent"].Value.split(
",")
location = messages.create_location()
location.latitude = lat
location.longitude = lng
location.address = addr
messages.insert_db(self.blued)
except Exception as e:
pass
elif msg_type == 5: # video
messages.type = 4
if rec["messageContent"].Value.find("||") != -1:
messages.media_path = rec[
"messageContent"].Value.split("||")[1]
else:
messages.media_path = rec["messageContent"].Value
elif msg_type == 6:
messages.content = rec["messageContent"].Value
elif msg_type == 10:
messages.content = rec["messageContent"].Value
elif msg_type == 11:
messages.type = 99
messages.content = rec["messageContent"].Value
elif msg_type == 12:
messages.type = 99
messages.content = rec["messageContent"].Value
elif msg_type == 13:
messages.type = 99
messages.content = rec["messageContent"].Value
elif msg_type == 14:
messages.type = 99
messages.content = rec["messageContent"].Value
elif msg_type == 24:
messages.type = 2
messages.content = "照片已销毁"
elif msg_type == 41: # 直播
try:
if "msgExtra" in rec and (
not rec["msgExtra"].IsDBNull):
data = json.loads(rec["msgExtra"].Value)
except Exception as e:
pass
# elif msg_type == 75: # live
# try:
# if "msgExtra" in rec and (not rec["msgExtra"].IsDBNull):
# data = json.loads(rec["msgExtra"].Value)
# except Exception as e:
# pass
elif msg_type == 55: # 撤回
messages.type = 99
messages.content = rec[
"fromUserName"].Value + " 撤回了该条消息。"
elif msg_type == 56: # card
try:
messages.type = 6
if "messageContent" in rec and (
not rec["messageContent"].IsDBNull):
data = json.loads(rec["messageContent"].Value)
if "avatar" in data:
messages.media_path = data["avatar"]
if "name" in data:
name = data["name"]
except Exception as e:
pass
elif msg_type == 58: # 直播
try:
if "messageContent" in rec and (
not rec["messageContent"].IsDBNull):
data = json.loads(rec["messageContent"].Value)
if "gif" in data:
messages.media_path = data["gif"]
except Exception as e:
pass
elif msg_type == 41: # live
try:
if "msgExtra" in rec and (
not rec["msgExtra"].IsDBNull):
data = json.loads(rec["msgExtra"].Value)
messages.content = rec["messageContent"].Value
if "avatar" in data:
messages.media_path = data["avatar"]
except Exception as e:
pass
elif msg_type == 67: # feed
try:
if "messageContent" in rec and (
not rec["messageContent"].IsDBNull):
data = json.loads(rec["msgExtra"].Value)
messages.content = rec["messageContent"].Value
feed = model_im.Feed()
if "feed_img_url" in data:
feed.image_path = data["feed_img_url"]
if "feed_text" in data:
feed.content = data["feed_text"]
if "feed_time" in data:
feed.send_time = convert_to_unixtime(
data["feed_time"])
feed.account_id = account_id
if "fromId" in rec and (
not rec["fromId"].IsDBNull):
feed.sender_id = rec["fromId"].Value
self.blued.db_insert_table_feed(feed)
except Exception as e:
pass
elif msg_type == 75: # live
try:
if "messageContent" in rec and (
not rec["messageContent"].IsDBNull):
data = json.loads(rec["msgExtra"].Value)
messages.content = rec["messageContent"].Value
if "avatar" in data:
messages.media_path = data["avatar"]
except Exception as e:
pass
except Exception as e:
print(e)
if messages.account_id and messages.talker_id and messages.sender_id:
self.blued.db_insert_table_message(messages)
self.blued.db_commit()
def parse_feed(self, node):
'''解析动态数据'''
print(self.comment_count)
userid = re.findall('\d+', node.PathWithMountPoint)[-1]
db = SQLiteParser.Database.FromNode(node, canceller)
if db is None:
return
ts = SQLiteParser.TableSignature('moments')
flag = 0
for rec in db.ReadTableRecords(ts, self.extractDeleted, True):
try:
if self._db_record_get_string_value(rec, '_id') == '':
continue
feed = model_im.Feed()
#多媒体数据
media_url = self.parse_media_from_blob(rec, 'media_c')
image_url = []
video_url = []
for url in media_url:
if re.findall('images', url):
image_url.append(url)
if re.findall('videos', url):
video_url.append(url)
#获赞数
likescount = self._db_record_get_int_value(rec, 'likes_count_c')
#动态id
feedpk = self._db_record_get_string_value(rec, 'id_c')
#评论数
comment_count = self.comment_count[feedpk] if feedpk in self.comment_count else 0
#动态时间
createtime = self._db_record_get_int_value(rec, 'createdTime_c')
#动态位置
if not IsDBNull(rec['location_c'].Value):
location = model_im.Location()
feed.location_id = location.location_id # 地址ID[INT]
loc = rec['location_c'].Value
if not IsDBNull(loc):
loc = Encoding.UTF8.GetString(loc)
else:
loc = ''
location.address = self.illegal_char(loc)
self.db_insert_table_location(location)
#发送者
senderid = self._db_record_get_int_value(rec, 'owner_c')
#动态文本
content = self._db_record_get_string_value(rec, 'value_c')
feed.account_id = userid # 账号ID[TEXT]
feed.sender_id = senderid # 发布者ID[TEXT]
feed.content = content # 文本[TEXT]
feed.image_path = ','.join(image_url) # 链接[TEXT]
feed.video_path = ','.join(video_url)
feed.send_time = createtime # 发布时间[INT]
feed.likecount = likescount # 赞数量[INT]
feed.commentcount = comment_count
feed.comment_id = feedpk
feed.source = node.AbsolutePath
feed.deleted = rec.IsDeleted
if feed.sender_id != 0:
self.db_insert_table_feed(feed)
except:
pass
self.db_commit()
def parse(self):
for aid in self.account:
m_dir = self.node.PathWithMountPoint
m_dir = os.path.join(m_dir, "Library/Caches/databases")
dl = os.listdir(m_dir)
a_dir = ""
for sub_dir in dl:
if sub_dir.__contains__(aid):
a_dir = sub_dir
break
if a_dir == "":
continue
db_id = os.listdir(os.path.join(
m_dir, a_dir))[0] # not with full path....
m_sql = "/Library/Caches/databases/{}/{}/twitter.db".format(
a_dir, db_id)
sub_node = self.node.GetByPath(m_sql)
if sub_node is None:
print('fatal error!')
continue
# add to dict
self.db_dict[aid] = m_sql
r_sql = sql.SQLiteConnection(
'Data Source = {}; ReadOnly = True'.format(
sub_node.PathWithMountPoint))
r_sql.Open()
cmd = sql.SQLiteCommand(r_sql)
cmd.CommandText = '''
select id, screenName, name, location, description from Users where screenName = '{}'
'''.format(aid)
reader = cmd.ExecuteReader()
if reader.Read():
a = model_im.Account()
a.account_id = GetInt64(reader, 0)
a.username = aid
current_id = GetInt64(reader, 0)
a.photo = ""
a.nickname = GetString(reader, 2)
a.address = GetString(reader, 3)
a.signature = GetString(reader, 4)
self.im.db_insert_table_account(a)
reader.Close()
cmd.CommandText = '''
select id, screenName, name, location, description from Users where screenName != '{}'
'''.format(aid)
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
raise IOError('E')
f = model_im.Friend()
f.account_id = current_id
f.friend_id = GetInt64(reader, 0)
f.remark = GetString(reader, 1)
f.address = GetString(reader, 3)
f.nickname = GetString(reader, 2)
f.signature = GetString(reader, 4)
self.im.db_insert_table_friend(f)
self.im.db_commit()
reader.Close()
cmd.CommandText = '''
select userId, text, date from Statuses
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
raise IOError('e')
blog = model_im.Feed()
blog.account_id = current_id
blog.content = GetString(reader, 1)
#blog.send_time = unity_c37r.format_mac_timestamp(unity_c37r.c_sharp_try_get_time(reader, 2))
blog.send_time = int(GetReal(reader, 2))
blog.sender_id = GetInt64(reader, 0)
self.im.db_insert_table_feed(blog)
self.im.db_commit()
#clean up
reader.Close()
cmd.Dispose()
r_sql.Close()
root_path = self.node.PathWithMountPoint
m_path = os.path.join(
root_path, 'Documents/com.atebits.tweetie.application-state')
talk_file = None
dirs = os.listdir(m_path)
for d in dirs:
res = self.check_app_account_id(d, aid)
if not res:
continue
talk_file = d
talk_node = self.node.GetByPath(
'/Documents/com.atebits.tweetie.application-state/{}'.
format(talk_file))
talk_file = talk_node.PathWithMountPoint # absolute path
print(talk_node.Size)
bp = BPReader(talk_node.Data).top
bp_arr = bp['$objects']
self.deserialize_message(bp_arr, aid, talk_file, current_id)
self.im.db_commit()
def parse(self):
app_node = self.node.GetByPath('Library/Caches/__xstore_app.db')
if app_node is None:
print('f****d')
return
conn = unity_c37r.create_connection(app_node.PathWithMountPoint)
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select nick, logo, gender from PUserInfo where userId = {}
'''.format(self.account)
reader = cmd.ExecuteReader()
if reader.Read():
a = model_im.Account()
a.account_id = self.account
a.nickname = unity_c37r.c_sharp_get_string(reader, 0)
a.photo = unity_c37r.c_sharp_get_string(reader, 1)
self.im.db_insert_table_account(a)
cmd.Dispose()
cmd.CommandText = '''
select userId, nick, logo, gender from PUserInfo where userId != {}
'''.format(self.account)
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
f = model_im.Friend()
f.account_id = self.account
f.friend_id = unity_c37r.c_sharp_get_long(reader, 0)
f.nickname = unity_c37r.c_sharp_get_string(reader, 1)
f.photo = unity_c37r.c_sharp_get_string(reader, 2)
self.im.db_insert_table_friend(f)
cmd.Dispose()
cmd.CommandText = '''
select pondId, adminUserId, pondName, pondLogo from PPondInfo
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
g = model_im.Chatroom()
g.account_id = self.account
g.chatroom_id = unity_c37r.c_sharp_get_long(reader, 0)
g.name = unity_c37r.c_sharp_get_string(reader, 2)
g.owner_id = unity_c37r.c_sharp_get_long(reader, 1)
g.photo = unity_c37r.c_sharp_get_string(reader, 3)
self.im.db_insert_table_chatroom(g)
cmd.Dispose()
cmd.CommandText = '''
select * from PItemInfo
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
feed = model_im.Feed()
feed.account_id = self.account
feed.attachment_link = unity_c37r.c_sharp_get_string(reader, 1)
feed.attachment_title = unity_c37r.c_sharp_get_string(reader, 4)
#feed.attachment_desc
feed.attachment_desc = '''price:{}\norigPrice:\n'''.format(unity_c37r.c_sharp_get_string(reader, 2), unity_c37r.c_sharp_get_string(reader, 3))
seller_id = unity_c37r.c_sharp_get_long(reader, 5)
seller_info = unity_c37r.c_sharp_get_long(reader, 6)
feed.sender_id = seller_id if seller_id != 0 else seller_info
self.im.db_insert_table_feed(feed)
cmd.Dispose()
conn.Close()
db_node = self.node.GetByPath('Library/Caches/__xstore_user_{}.db'.format(self.account))
if db_node is None:
print('f****d')
return
conn = unity_c37r.create_connection(db_node.PathWithMountPoint)
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select Sid, Uid, content, extJson, timeStamp from PMessage
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
m = model_im.Message()
m.account_id = self.account
m.sender_id = unity_c37r.c_sharp_get_long(reader, 1)
m.talker_id = unity_c37r.c_sharp_get_long(reader, 0)
m.send_time = unity_c37r.c_sharp_get_long(reader, 4) / 1000
try:
string = unity_c37r.c_sharp_get_string(reader, 2)
js = json.loads(string)
tp = js.get('contentType')
if tp == 1:
m.content = js.get('text').get('text')
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 17: # trade...
#m.content = js.get('content')
m.content = js.get('title')
deal = model_im.Deal()
m.type = model_im.MESSAGE_CONTENT_TYPE_RECEIPT
m.extra_id = deal.deal_id
deal.money = js.get('content')
deal.description = js.get('title')
self.im.db_insert_table_deal(deal)
elif tp == 10:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
js = js.get('actionCard')
m.content = '''title:{}\ncontent:{}'''.format(js.get('memo'), js.get('title'))
elif tp == 2: # image
m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
m.media_path = js.get('image').get('pics')[0].get('url')
elif tp == 8:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''title:{}\ncontent:{}'''.format(js.get('imageCard').get('title'), js.get('imageCard').get('content'))
elif tp == 6:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
#m.content = '''title:{}\ncontent:{}\nstring:{}'''.format(js.get('textCard').get('title'), js.get('textCard').get('memo'), string)
js = js.get('textCard')
title = js.get('title')
# if title is None:
# m.content = '''content:{}'''.format(js.get('content'))
# else:
# m.content = '''title:{}\ncontent:{}\nstring:{}'''.format(js.get('title'), js.get('memo'), string)
content = js.get('content')
memo = js.get('memo')
m.content = '''title:{}\ncontent:{}\nmemo:{}'''.format(title, content, memo)
elif tp == 16:
m.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT
m.content = js.get('itemCard').get('title')
deal = model_im.Deal()
js = js.get('itemCard').get('item')
deal.deal_id = js.get('itemId')
m.extra_id = deal.deal_id
m.media_path = js.get('mainPic')
deal.description = js.get('title')
deal.money = js.get('price')
self.im.db_insert_table_deal(deal)
elif tp == 11: # 留言
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''title:{}\ncontent:{}'''.format(js.get('reply').get('title'), js.get('reply').get('content'))
# url is lost... waiting for support.
elif tp == 20:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = js.get('text')
elif tp == 7:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = js.get('itemCard').get('title')
elif tp == 14:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = js.get('tip').get('tips')
else:
print('detect unsupported type:%d' % tp)
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = string
self.im.db_insert_table_message(m)
except Exception as e:
print 'find error string:{}'.format(string)
logging.error(e)
self.im.db_commit()
cmd.Dispose()
cmd.CommandText = '''
select XSummary, Session$$$$sessionId , ts, Sender from XMessageCenterItem
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
m = model_im.Message()
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = unity_c37r.c_sharp_get_string(reader, 0)
m.talker_id = unity_c37r.c_sharp_get_long(reader, 1)
m.sender_id = unity_c37r.c_sharp_get_long(reader, 3)
m.send_time = unity_c37r.c_sharp_get_long(reader, 2) / 1000
self.im.db_insert_table_message(m)
self.im.db_commit()
cmd.Dispose()
conn.Close()
def parse(self):
for i in self.result_sql:
current_id = None
connection = sql.SQLiteConnection(
'Data Source = {}; ReadOnly=True'.format(i))
connection.Open()
cmd = sql.SQLiteCommand(connection)
cmd.CommandText = '''
select uid, nick, avatarMediaId, mobile, gender, birthdayValue, address, extension, email from contact
'''
reader = cmd.ExecuteReader()
idx = 0
f_dict = dict()
while reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError("f****d")
if idx == 0:
a = model_im.Account()
a.account_id = GetInt64(reader, 0)
a.nickname = GetString(reader, 1)
# real_name????
# photo ... pass for a while.
self.get_picture(GetString(reader, 2), AVATAR)
a.telephone = GetString(reader, 3)
a.gender = GetInt64(reader, 4)
a.birthday = GetInt64(reader, 5)
a.address = GetString(reader, 6)
a.email = GetString(reader, 8)
r = GetString(reader, 7)
if r is '':
pass
else:
r = json.loads(r)
if r.get('ownness') is not None:
#r = r.get('ownness').get(status)
pass
else:
pass
self.im.db_insert_table_account(a)
idx += 1
current_id = a.account_id
continue
f = model_im.Friend()
f.friend_id = GetInt64(reader, 0)
f.nickname = GetString(reader, 1)
# photo....
# f.photo = #TODO generate photo id....
f.photo = self.get_picture(GetString(reader, 2), AVATAR)
f.account_id = current_id
f.telephone = GetString(reader, 3)
f.gender = GetInt64(reader, 4)
f.birthday = GetInt64(reader, 5)
f.deleted = 0
f.email = GetString(reader, 8)
f.address = GetString(reader, 6)
f.type = model_im.FRIEND_TYPE_FRIEND
r = GetString(reader, 7)
if r is '':
pass
else:
r = json.loads(r)
if r.get('ownness') is not None:
pass #TODO fix it later
self.im.db_insert_table_friend(f)
f_dict[f.friend_id] = f
cmd.Dispose()
cmd.CommandText = '''
select conversationId, title, createdAt, extensionJson, ownerId,
memberLimit, memberCount, automaticIcon, customIcon from WKConversation where conversationType != 2
'''
reader = cmd.ExecuteReader()
groups = list()
while reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
g = model_im.Chatroom()
g.account_id = current_id
g.chatroom_id = GetString(reader, 0)
g.name = GetString(reader, 1)
g.photo = self.get_picture(GetString(reader, 8), AVATAR)
g.create_time = GetInt64(reader, 2) / 1000
g.owner_id = GetInt64(reader, 4)
g.max_member_count = GetInt64(reader, 5)
g.member_count = GetInt64(reader, 6)
groups.append(g.chatroom_id)
self.im.db_insert_table_chatroom(g)
chat_tbl_list = list()
cmd.Dispose()
cmd.CommandText = '''
select tbl_name from sqlite_master where tbl_name like 'WKChat_%' and type = 'table'
'''
reader = cmd.ExecuteReader()
while reader.Read():
r = GetString(reader, 0)
if r is '' or r.__contains__('fts'):
continue
chat_tbl_list.append(r)
cmd.Dispose()
for r in chat_tbl_list:
cmd.CommandText = '''
select messageId, conversationId, localSentTime, content, attachmentsType, senderId,
attachmentsJson from {}
'''.format(r)
reader = cmd.ExecuteReader()
while reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
msg = model_im.Message()
msg.deleted = 0
msg.account_id = current_id
cv_id = GetString(reader, 1)
# update conversation id
if cv_id.__contains__(':'):
cvl = cv_id.split(':')
if int(cvl[0]) == current_id:
msg.talker_id = cvl[1]
else:
msg.talker_id = cvl[0]
msg.talker_type = model_im.CHAT_TYPE_FRIEND
else:
msg.talker_id = cv_id
msg.talker_type = model_im.CHAT_TYPE_GROUP
msg.sender_id = GetInt64(reader, 5)
msg.is_sender = 1 if msg.sender_id == current_id else 0
msg.content = GetString(reader, 3)
msg.send_time = GetInt64(reader, 2)
msg.msg_id = GetInt64(reader, 0)
msg.source = i
#TODO add other message decryptor.... and parse etc.
tp = GetInt64(reader, 4)
try:
if tp == 1 or tp == 500 or tp == 501:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 1101:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
msg.content = '[call message]'
elif tp == 600:
msg.type = model_im.MESSAGE_CONTENT_TYPE_CONTACT_CARD
string = GetString(reader, 6)
js = json.loads(string)
name = js.get('attachments')[0].get(
'extension').get('name')
uid = js.get('attachments')[0].get(
'extension').get('uid')
msg.content = 'uid:{}\nname:{}'.format(uid, name)
elif tp == 102:
msg.type = model_im.MESSAGE_CONTENT_TYPE_CONTACT_CARD
string = GetString(reader, 6)
js = json.loads(string)
title = js.get('attachments')[0].get(
'extension').get('title')
text = js.get('attachments')[0].get(
'extension').get('text')
pic = js.get('attachments')[0].get(
'extension').get('picUrl')
msg.content = 'title:{}\ntext:{}\npicUrl:{}'.format(
title, text, pic)
elif tp == 202 or tp == 103:
msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO
string = GetString(reader, 6)
js = json.loads(string)
media_id = js.get('attachments')[0].get(
'extension').get('picUrl')
abs_path = self.root.PathWithMountPoint
f_name = os.path.join(
abs_path,
"Library/Caches/videoCaches/%s.mp4" % media_id)
if os.path.exists(f_name):
msg.media_path = f_name
else:
msg.content = "video message: not cached"
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 104: # location ?
string = GetString(reader, 6)
js = json.loads(string)
lati = js.get('attachments')[0].get(
'extension').get('latitude')
lng = js.get('attachments')[0].get(
'extension').get('longitude')
name = js.get('attachments')[0].get(
'extension').get('locationName')
msg.location = md5(str(lati) + str(lng))
msg.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION
msg.content = name
l = model_im.Location()
l.location_id = msg.location
l.deleted = 0
l.latitude = lati
l.longitude = lng
l.address = name
l.timestamp = GetInt64(reader, 2)
self.im.db_insert_table_location(l)
elif tp == 2: # image
msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
string = GetString(reader, 6)
js = json.loads(string)
media_id = js.get('photoContent').get('mediaId')
t_f_name = js.get('photoContent').get('filename')
if t_f_name is None or t_f_name == "":
#print(string)
ext = ''
else:
fn, ext = os.path.splitext(t_f_name)
if ext == "JPG" or ext == "jpg" or ext == 'MOV' or ext == 'mov':
fn = self.get_picture(media_id, JPG)
elif ext == 'PNG' or ext == 'png':
fn = self.get_picture(media_id, PNG)
elif ext == 'gif' or ext == 'GIF':
fn = self.get_picture(media_id, GIF)
else:
fn = self.get_picture(media_id, SYSTEM)
if fn != "":
msg.media_path = fn
else:
msg.content = 'image message not cached...'
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
#TODO Fix it after rp is certain...
elif tp == 900 or tp == 901:
msg.content = GetString(reader, 6)
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
else:
msg.content = GetString(reader, 3) if GetString(
reader, 3) is not '' else GetString(reader, 6)
msg.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM
except:
self.log_print('error occurs: {}'.format(
GetString(reader, 3)))
continue
self.im.db_insert_table_message(msg)
cmd.Dispose()
# group members....
for g in groups:
members = list()
for t in chat_tbl_list:
cmd.CommandText = '''
select senderId from {} where conversationId = '{}' group by senderId
'''.format(t, g)
reader = cmd.ExecuteReader()
while reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
m_id = GetInt64(reader, 0)
if members.__contains__(m_id) or m_id == current_id:
continue
members.append(m_id)
cmd.Dispose()
for m in members:
cm = model_im.ChatroomMember()
cm.account_id = current_id
cm.member_id = m
if f_dict.__contains__(m):
#f = model_im.Friend()
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
f = f_dict[m]
cm.address = f.address
cm.display_name = f.nickname
cm.birthday = f.birthday
cm.photo = f.photo
cm.deleted = 0
cm.gender = f.gender
cm.email = f.email
cm.telephone = f.telephone
cm.signature = f.signature
cm.chatroom_id = g
self.im.db_insert_table_chatroom_member(cm)
self.im.db_commit()
#self.im.db_close()
#以下部分不兼容老版本分析:
cmd.CommandText = '''
select dingId, senderUid, sendAt, content from Ding
'''
reader = cmd.ExecuteReader()
feed_dict = dict()
while reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
s_id = GetInt64(reader, 1)
d_id = GetInt64(reader, 0)
s_time = GetInt64(reader, 2) / 1000
s_content = GetString(reader, 3)
feed = model_im.Feed()
feed.sender_id = s_id
feed.account_id = current_id
feed.repeated = 0
feed.send_time = s_time
feed.content = s_content
feed_dict[d_id] = feed
#feed.comments = s_id
#self.im.db_insert_table_feed(feed)
cmd.Dispose()
cmd.CommandText = '''
select dingId, commentId, commenterUid, attachmentJSON, createAt from dingcomment
'''
try:
reader = cmd.ExecuteReader()
except:
reader = None
while reader is not None and reader.Read():
# if canceller.IsCancellationRequested:
# self.im.db_close()
# raise IOError('f****d')
fcm = model_im.FeedComment()
d_id = GetInt64(reader, 0)
if not feed_dict.__contains__(d_id):
continue
feed_dict[d_id].comments = str()
feed_dict[d_id].comments += '{},'.format(GetInt64(reader, 1))
fcm.comment_id = GetInt64(reader, 1)
string = GetString(reader, 3)
try:
fcm.content = json.loads(string).get('text')
fcm.sender_id = GetInt64(reader, 1)
fcm.sender_name = "" if not f_dict.__contains__(
fcm.sender_id) else f_dict[fcm.sender_id].nickname
fcm.create_time = GetInt64(reader, 4) / 1000
except:
pass
self.im.db_insert_table_feed_comment(fcm)
for k in feed_dict:
self.im.db_insert_table_feed(feed_dict[k])
feed_dict.clear() # no longer use...
cmd.Dispose()
# 出勤活动
cmd.CommandText = '''
select value from WebPersistenceModel where key like 'fastCheck_%' or 'asyncCheck_%'
'''
try:
reader = cmd.ExecuteReader()
except:
reader = None
while reader is not None and reader.Read():
try:
string = GetString(reader, 0)
js = json.loads(string)
feed = model_im.Feed()
feed.sender_id = current_id
addr = js.get('checkResult').get('address')
time = js.get('checkResult').get('checkTime')
f_id = js.get('checkResult').get('id')
method = js.get('checkResult').get('locationMethod')
s_type = js.get('checkResult').get('scheduleType')
feed.content = '''
event:{}
address:{}
method:{}
'''.format(s_type, addr, method)
feed.send_time = time / 1000
self.im.db_insert_table_feed(feed)
except:
continue
self.im.db_commit()
cmd.Dispose()
connection.Close()
try:
self.parse_recovery(i, current_id, chat_tbl_list)
except:
pass
def _get_user_feed(self, node):
account_id = node.Name[12:-4]
with open(node.PathWithMountPoint, "r") as f:
json_data = json.loads(f.read())
if "list" in json_data:
feed_lists = json_data["list"]
for item in feed_lists:
feed = model_im.Feed()
feed.source = node.AbsolutePath
feed.account_id = account_id
if "userAccid" in item:
feed.sender_id = item["userAccid"]
if "createTime" in item:
feed.send_time = convert_to_unixtime(
item["createTime"])
if "text" in item:
feed.content = item["text"]
if "liked" in item and item["liked"] == True:
like_lists = item["likeList"]
for item_like in like_lists:
f1 = feed.create_like()
f1.send_id = item_like["userAccid"]
f1.sender_name = item_like["name"]
f1.create_time = convert_to_unixtime(
item["createTime"])
feed.likecount += 1
if feed.likecount == 0:
feed.like_id = 0
if "location" in item:
loc = feed.create_location()
loc.address = item["location"]
if "longitude" in item:
loc.longitude = item["longitude"]
if "latitude" in item:
loc.longitude = item["latitude"]
loc.timestamp = convert_to_unixtime(item["createTime"])
if "commentList" in item:
comment_list = item["commentList"]
if comment_list:
for item_cmt in comment_list:
fc = feed.create_comment()
fc.sender_id = item_cmt["userAccid"]
fc.sender_name = item_cmt["sponsorName"]
fc.content = item_cmt["text"]
fc.create_time = convert_to_unixtime(
item_cmt["createTime"])
feed.commentcount += 1
if feed.commentcount == 0:
feed.comment_id = 0
if item["feedType"] != "text":
resource_lists = item["images"]
if resource_lists:
urls = []
for item_res in resource_lists:
urls.append(item_res)
if len(urls) > 0:
if item["feedType"] == "image":
feed.image_path = ','.join(
str(u) for u in urls)
else:
feed.video_path = ','.join(
str(u) for u in urls)
feed.insert_db(self.bulletMessage)
self.bulletMessage.db_commit()
def parse_feed(self, dbPath):
'''解析动态数据'''
db = SQLite.SQLiteConnection(
'Data Source = {}; ReadOnly = True'.format(dbPath))
db.Open()
db_cmd = SQLite.SQLiteCommand(db)
try:
if self.db is None:
return
db_cmd.CommandText = '''select a.item_id, a.mediaBytesInfo, a.timestamp, a.user_id,
group_concat(b."action") as "action", group_concat(b.comment) as comment,
group_concat(b.timestamp) as comment_time, group_concat(b.user_id) as comment_user_id, group_concat(b.commend_id) as comment_id, a.deleted
from bb_dl_item_info as a left join bb_dl_comment_info as b on a.item_id = b.item_id group by a.item_id'''
sr = db_cmd.ExecuteReader()
while (sr.Read()):
try:
feed = model_im.Feed()
if canceller.IsCancellationRequested:
break
feed.account_id = self.account_id
feed.sender_id = self._db_reader_get_int_value(sr, 3)
feed.send_time = self._db_reader_get_int_value(sr, 2)
location_value = '' #location_value用于保存提取出的位置信息的值
img_name = [] #img_name列表用于保存图片名称
text_content = '' #text_content用于保存提取出的正文内容
try:
value = str(self._db_reader_get_blob_value(
sr, 1)).encode('hex')
data = [] #data用于存储格式化的字节信息
for i, item in enumerate(value): #生成data列表
if i % 2 == 0:
data.append(value[i] + value[i + 1])
else:
pass
for i, d in enumerate(data): #判断与获取位置信息
if d == '42' and data[i + 1] == '4a':
location_start = i + 3
location_lens = int(data[i + 2], 16)
location_end = location_start + location_lens
location_value = value[
location_start * 2:location_end *
2:].decode('hex').decode('utf-8')
if d == '1a' and data[i + 2] == '0a':
img_start = i + 4
img_lens = int(data[i + 3], 16)
img_end = img_start + img_lens
img_name.append(
value[img_start * 2:img_end *
2:].decode('hex').decode('utf-8'))
if d == '22':
text_start = i + 2
text_lens = int(data[i + 1], 16)
text_end = text_start + text_lens
text_content = value[text_start * 2:text_end *
2:].decode('hex').decode(
'utf-8')
if location_value is not '':
location = model_im.Location()
feed.location_id = location.location_id
location.address = location_value
self.db_insert_table_location(location)
feed.content = text_content
image_name = []
for image in img_name:
nodes = self.cacheNode.Search('/' + image +
'.*\..*$')
for node in nodes:
image_name.append(node.AbsolutePath)
feed.image_path = ','.join(image_name)
except:
pass
actions = self._db_reader_get_string_value(sr,
4).split(',')
comment_id = self._db_reader_get_string_value(sr,
8).split(',')
likes = []
comments = []
like_count = 0
comment_count = 0
for i, action in enumerate(actions):
if action == 'like':
like_count += 1
likes.append(comment_id[i])
elif not IsDBNull(action):
comment_count += 1
comments.append(comment_id[i])
feed.likes = ','.join(likes)
feed.likecount = like_count
feed.comments = ','.join(comment_id)
feed.commentcount = comment_count
feed.deleted = self._db_reader_get_int_value(sr, 9)
if feed.content is not None and feed.image_path is not None:
self.db_insert_table_feed(feed)
except:
traceback.print_exc()
sr.Close()
self.db_commit()
db_cmd.Dispose()
db.Close()
except Exception as e:
print(e)
def generate_model(self, bplist_dic):
try:
global fid
if 'data' not in bplist_dic:
return []
elif bplist_dic['data'] is '':
return []
data = bplist_dic['data']
#获取用户数据（设置为系统消息）
if not isinstance(data, dict):
return []
if 'agent_title' in data:
agent_title = self.verify_dic(data, 'agent_title')
regist_time = self.verify_dic(data, 'reg_time')
province = self.verify_dic(data, 'province')
city = self.verify_dic(data, 'city')
address = self.verify_dic(data, 'address')
area = self.verify_dic(data, 'area')
agent_money = self.verify_dic(data, 'agent_money')
nickname = self.verify_dic(data, 'nickname')
username = self.verify_dic(data, 'username')
telecode = self.verify_dic(data, 'telecode')
mobile = self.verify_dic(data, 'mobile')
sex = self.verify_dic(data, 'sex')
parent_nickname = self.verify_dic(data, 'parent_nickname')
parent_name = self.verify_dic(data, 'parent_name')
parent_telecode = self.verify_dic(data, 'parent_telecode')
parent_mobile = self.verify_dic(data, 'parent_mobile')
friend = model_im.Friend()
friend.account_id = self.uid
friend.nickname = nickname
friend.fullname = username
friend.telephone = mobile
friend.address = province + city + address
friend.signature = '代理头衔：' + agent_title + ' 代理金额：' + agent_money + ' 上级代理人:' + parent_name + ' 上级代理人联系方式：' + parent_telecode + parent_mobile
self.db_insert_table_friend(friend)
#推荐消息
elif 'share_url' in data:
desc = self.verify_dic(data, 'desc')
seller_name = self.verify_dic(data, 'seller_name')
logo = self.verify_dic(data, 'logo')
share_url = self.verify_dic(data, 'share_url')
title = self.verify_dic(data, 'title')
message = model_im.Message()
message.account_id = self.uid
message.talker_id = 2
message.talker_name = '未知'
message.sender_id = 1
message.is_sender = 1
message.type = model_im.FAVORITE_TYPE_LINK
message.talker_type = model_im.CHAT_TYPE_FRIEND
link = model_im.Link()
message.link_id = link.link_id
link.title = title
link.url = share_url
link.content = seller_name + ':' + desc
link.image = logo
self.db_insert_table_link(link)
self.db_insert_table_message(message)
#刷新好友[设置为系统消息]
elif 'login' in data:
name = self.verify_dic(data, 'name')
group_level = self.verify_dic(data, 'group_level')
username = self.verify_dic(data, 'username')
author_img = self.verify_dic(data, 'author_img')
pnickname = self.verify_dic(data, 'pnickname')
group = self.verify_dic(data, 'group')
sex = self.verify_dic(data, 'sex')
puid = self.verify_dic(data, 'puid')
province = self.verify_dic(data, 'province')
signature = self.verify_dic(data, 'signature')
address = self.verify_dic(data, 'address')
id_number = self.verify_dic(data, 'id_number')
usable_money = self.verify_dic(data, 'usable_money')
telcode = self.verify_dic(data, 'telcode')
last_login_time = self.verify_dic(data, 'last_login_time')
avatar = self.verify_dic(data, 'avatar')
reg_time = self.verify_dic(data, 'reg_time')
city = self.verify_dic(data, 'city')
qq = self.verify_dic(data, 'qq')
email = self.verify_dic(data, 'last_login_ip')
nickname = self.verify_dic(data, 'nickname')
pusername = self.verify_dic(data, 'pusername')
mobile = self.verify_dic(data, 'mobile')
wechat = self.verify_dic(data, 'wechat')
friend = model_im.Friend()
friend.account_id = self.uid
friend.nickname = nickname
friend.fullname = username
friend.photo = avatar
friend.gender = sex
friend.telephone = mobile
friend.email = email
friend.address = province + city + address
friend.signature = '上级代理人：' + pnickname + ' 所属群组：' + group + ' 身份证号：' + id_number + ' 注册时间:' + reg_time + ' qq：' + qq + ' 微信：' + wechat + ' 个性签名：' + signature
self.db_insert_table_friend(friend)
elif 'data' in data:
if not isinstance(data['data'], dict):
return []
if 'key0' in data['data']:
#推荐成员
if 'layer_title' in data['data']['key0']:
data = data['data']['key0']
layer_title = self.verify_dic(data, 'layer_title')
uid = self.verify_dic(data, 'uid')
avatar = self.verify_dic(data, 'avatar')
sex = self.verify_dic(data, 'sex')
name = self.verify_dic(data, 'name')
reg_time = self.verify_dic(data, 'reg_time')
group_icon = self.verify_dic(data, 'group_icon')
group_title = self.verify_dic(data, 'group_title')
nickname = self.verify_dic(data, 'nickname')
message = model_im.Message()
message.account_id = self.uid
message.talker_id = 2
message.talker_name = '未知'
message.is_sender = 1
message.type = model_im.MESSAGE_CONTENT_TYPE_LINK
message.content = '推荐成员\n' + 'id:' + uid + '\n姓名：' + name + '\n注册时间：' + reg_time + '头衔：' + layer_title
message.talker_type = model_im.CHAT_TYPE_FRIEND
self.db_insert_table_message(message)
friend = model_im.Friend()
friend.account_id = self.uid
friend.nickname = nickname
friend.fullname = name
friend.photo = avatar
friend.gender = sex
self.db_insert_table_friend(friend)
#动态
elif 'islike' in data['data']['key0']:
dat = data['data']
for key in dat.keys():
data = dat[key]
view = self.verify_dic(data, 'view')
comment_list = self.verify_dic(
data, 'comment_list')
create_time = self.verify_dic(data, 'create_time')
id = self.verify_dic(data, 'id')
description = self.verify_dic(data, 'description')
content = self.verify_dic(data, 'content')
video_url = self.verify_dic(data, 'video_url')
comment_count = self.verify_dic(
data, 'comment_count')
avatar = self.verify_dic(data, 'avatar')
islike = self.verify_dic(data, 'islike')
photo_url = []
if 'key0' in data['photo_url']:
for key in data['photo_url'].keys():
photo_url.append(data['photo_url'][key])
title = self.verify_dic(data, 'title')
uid = self.verify_dic(data, 'uid')
likes = self.verify_dic(data, 'likes')
share = self.verify_dic(data, 'share')
name = self.verify_dic(data, 'name')
friend = model_im.Friend()
friend.account_id = self.uid
friend.friend_id = uid
friend.nickname = name
friend.fullname = name
friend.photo = avatar
feed = model_im.Feed()
feed.account_id = self.uid
feed.sender_id = uid
feed.content = content
feed.image_path = ','.join(photo_url)
feed.video_path = video_url
if create_time is not '':
feed.send_time = int(
time.mktime(
time.strptime(create_time,
"%Y-%m-%d %H:%M:%S")))
feed.likecount = likes
feed.rtcount = share
feed.commentcount = comment_count
self.db_insert_table_friend(friend)
self.db_insert_table_feed(feed)
#获取群组信息
elif 'group_id' in data['data']['key0']:
dat = data['data']
for key in dat.keys():
title = self.verify_dic(data, 'title')
group_id = self.verify_dic(data, 'group_id')
level = self.verify_dic(data, 'level')
if group_id is '':
return
chatroom = model_im.Chatroom()
chatroom.account_id = self.uid
chatroom.chatroom_id = group_id
chatroom.name = title
self.db_insert_table_chatroom(chatroom)
#获取位置信息
elif 'obj' in data['data']:
city = self.verify_dic(data, 'city')
area = self.verify_dic(data, 'area')
personalName = self.verify_dic(data, 'personalName')
site = self.verify_dic(data, 'site')
province = self.verify_dic(data, 'province')
location = province + city + area + site + personalName
#广告（系统）
elif 'showimg' in data:
imgurl = self.verify_dic(data, 'imgurl')
url = self.verify_dic(data, 'url')
message = model_im.Message()
message.account_id = self.uid
message.talker_id = 2
message.talker_name = '未知'
message.sender_id = 2
message.is_sender = 0
message.type = model_im.FAVORITE_TYPE_LINK
message.talker_type = model_im.CHAT_TYPE_FRIEND
link = model_im.Link()
message.link_id = link.link_id
link.url = url
link.image = imgurl
self.db_insert_table_link(link)
self.db_insert_table_message(message)
#获取帮助信息
elif 'url' in data:
url = self.verify_dic(data, 'url')
elif isinstance(data, str):
#获取认证信息
if re.findall('author', str(data)):
authentification_url = data
#获取下载信息
elif re.findall('download', str(data)):
download_url = data
self.db_commit()
except:
traceback.print_exc()
def parse_feed(self, node):
'''解析动态数据'''
userid = re.findall('\d+', node.PathWithMountPoint)[-1]
db = SQLiteParser.Database.FromNode(node, canceller)
if db is None:
return
try:
ts = SQLiteParser.TableSignature('Moment')
self.sticker = {}
for rec in db.ReadTableRecords(ts, self.extractDeleted, True):
try:
if self._db_record_get_string_value(rec, 'objectID') == '':
continue
feed = model_im.Feed()
#多媒体数据
media_url = ''
tree = BPReader.GetTree(
MemoryRange.FromBytes(rec['mediaDictionaries'].Value))
if tree is None:
break
media_json = self.bplist2json(tree)
if media_json is not None:
media_url = media_json['key']['url']
#获赞数
likescount = self._db_record_get_int_value(
rec, 'likesCount')
#评论数
commentcount = self._db_record_get_int_value(
rec, 'commentsCount')
#动态id
feedpk = self._db_record_get_int_value(rec, 'primaryKeyID')
#动态时间
createtime = self._db_record_get_int_value(
rec, 'createdTime')
#动态位置
location = self._db_record_get_blob_value(
rec, 'locationDictionary')
if location is not None:
location = json.loads(location)
#location = Encoding.UTF8.GetString(rec['locationDictionary'].Value) if not IsDBNull(rec['locationDictionary'].Value) else None
if location is not None:
try:
coordinates = location['coordinates']
except:
coordinates = ''
latitude = 0
longitude = 0
address = ''
if coordinates is not '':
latitude = coordinates[0]
longitude = coordinates[1]
address = location['address'] + location['name']
location = model_im.Location()
feed.location_id = location.location_id # 地址ID[INT]
location.latitude = latitude
location.longitude = longitude
location.address = address
self.db_insert_table_location(location)
#发送者
senderid = self._db_record_get_int_value(
rec, 'owner_primaryKeyID')
if senderid == -1:
senderid = -2
#是否给该条动态点赞
isliked = self._db_record_get_int_value(rec, 'haveLiked')
#动态文本
content = self._db_record_get_string_value(rec, 'value')
feed.account_id = userid # 账号ID[TEXT]
feed.sender_id = senderid # 发布者ID[TEXT]
feed.content = content # 文本[TEXT]
if re.findall("image", media_url):
feed.image_path = media_url
elif re.findall("video", media_url):
feed.video_path = media_url
feed.send_time = createtime # 发布时间[INT]
feed.likecount = likescount # 赞数量[INT]
feed.commentcount = commentcount # 评论数量[INT]
feed.comment_id = feedpk
feed.deleted = rec.IsDeleted
self.db_insert_table_feed(feed)
except:
pass
self.db_commit()
except:
pass
def parse(self, aid):
db_node = self.node.GetByPath('databases/{}'.format(
self.account_dbs[aid]))
if db_node is None:
return
conn = unity_c37r.create_connection_tentatively(db_node)
cmd = sql.SQLiteCommand(conn)
f_dict = dict()
cmd.CommandText = '''
select user_id, username, name, description, web_url, image_url from users where user_id = {}
'''.format(aid)
reader = cmd.ExecuteReader()
if reader.Read():
a = model_im.Account()
a.account_id = unity_c37r.c_sharp_get_long(reader, 0)
a.nickname = unity_c37r.c_sharp_get_string(reader, 2)
a.username = unity_c37r.c_sharp_get_string(reader, 1)
a.signature = parse_description(
unity_c37r.c_sharp_get_blob(reader, 3))
a.photo = unity_c37r.c_sharp_get_string(reader, 5)
f_dict[a.account_id] = a
self.im.db_insert_table_account(a)
reader.Close()
cmd.CommandText = '''
select user_id, username, name, description, web_url, image_url from users where user_id != {}
'''.format(aid)
reader = cmd.ExecuteReader()
while reader.Read():
f = model_im.Friend()
f.account_id = aid
f.friend_id = unity_c37r.c_sharp_get_long(reader, 0)
f.nickname = unity_c37r.c_sharp_get_string(reader, 2)
f.remark = unity_c37r.c_sharp_get_string(reader, 1)
f.signature = parse_description(
unity_c37r.c_sharp_get_blob(reader, 3))
f.photo = unity_c37r.c_sharp_get_string(reader, 5)
f_dict[f.account_id] = f
self.im.db_insert_table_friend(f)
reader.Close()
cmd.CommandText = '''
select status_id, author_id, content, created, favorite_count, retweet_count from statuses
'''
reader = cmd.ExecuteReader()
while reader.Read():
feed = model_im.Feed()
feed.account_id = aid
feed.likecount = unity_c37r.c_sharp_get_long(reader, 4)
feed.rtcount = unity_c37r.c_sharp_get_long(reader, 5)
feed.content = parse_statuses(
unity_c37r.c_sharp_get_blob(reader, 2))
feed.sender_id = unity_c37r.c_sharp_get_long(reader, 1)
feed.send_time = unity_c37r.c_sharp_get_long(reader, 3) / 1000
self.im.db_insert_table_feed(feed)
reader.Close()
cmd.CommandText = '''
select user_id, created, data, conversation_id from conversation_entries
'''
reader = cmd.ExecuteReader()
while reader.Read():
cid = unity_c37r.c_sharp_get_string(reader, 3)
uids = cid.split('-')
obj_id = None
if uids[0] == str(aid):
obj_id = uids[1]
else:
obj_id = uids[0]
m = model_im.Message()
m.account_id = aid
m.send_time = unity_c37r.c_sharp_get_long(reader, 1) / 1000
m.content = parse_messages(unity_c37r.c_sharp_get_blob(reader, 2))
m.talker_id = obj_id
m.send_id = unity_c37r.c_sharp_get_long(reader, 0)
if f_dict.__contains__(int(obj_id)):
m.talker_name = f_dict[int(obj_id)].nickname
self.im.db_insert_table_message(m)
self.im.db_commit()
self.parse_recovery(aid)
def parse(self, aid):
files = self.cache + '/{}.db'.format(aid)
if not os.path.exists(self.cache + '/{}.db'.format(aid)):
self.m_print('no sql file,parse exit!')
return
conn = sql.SQLiteConnection(
'Data Source = {}; Readonly = True'.format(self.cache +
'/{}.db'.format(aid)))
conn.Open()
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select iconMedia, nick, gender, city, mobile, real_name, dingtalkId, email, activeTime
from tbuser where uid = {}
'''.format(aid)
reader = cmd.ExecuteReader()
reader.Read()
a = model_im.Account()
a.account_id = aid
a.nickname = GetString(reader, 1)
if GetString(reader, 2) == 'F':
a.gender = model_im.GENDER_FEMALE
elif GetString(reader, 2) == 'M':
a.gender = model_im.GENDER_MALE
else:
a.gender = model_im.GENDER_OTHER
a.address = GetString(reader, 3)
a.telephone = GetString(reader, 4)
a.email = GetString(reader, 7)
a.photo = GetString(reader, 0)
self.im.db_insert_table_account(a)
cmd.Dispose()
cmd.CommandText = '''
select uid, iconMedia, nick, gender, city, mobile, email, extensation from tbuser
where uid != {}
'''.format(aid)
reader = cmd.ExecuteReader()
f_dict = dict()
while reader.Read():
f = model_im.Friend()
f.account_id = aid
f.friend_id = GetInt64(reader, 0)
f.photo = GetString(reader, 1)
gender = GetString(reader, 3)
if gender == 'F':
f.gender = model_im.GENDER_FEMALE
elif gender == 'M':
f.gender = model_im.GENDER_MALE
else:
f.gender = model_im.GENDER_OTHER
f.nickname = Getstring(reader, 2)
f.address = GetString(reader, 4)
f.telephone = GetString(reader, 5)
f.email = GetString(reader, 6)
f.source = files
ext = Getstring(reaser, 7)
js = json.loads(ext)
if js.get('ownness') is not None:
f.signature = js.get('ownness')[0].get('status')
#self.im.db_insert_table_friend(f)
f_dict[f.friend_id] = f
cmd.Dispose()
# 暂时未加入分组信息
cmd.CommandText = '''
select uid, empName, orgName, depName from tb_user_intimacy
'''
reader = cmd.ExecuteReader()
while reader.Read():
try:
#f = model_im.Friend()
f_dict[GetInt64(reader, 0)].remark = GetString(reader, 2)
except:
continue
for k in f_dict:
self.im.db_insert_table_friend(f_dict[k])
cmd.Dispose()
cmd.CommandText = '''
select tbdingcontent.content, tbdingcontent.dingId, tbdinglist.senderId,
tbdinglist.dingCreatedAt, tbdinglist.latestComments
from tbdingcontent,tbdinglist where tbdingcontent.dingId = tbdinglist.dingId
'''
reader = cmd.ExecuteReader()
while reader.Read():
feed = model_im.Feed()
feed.account_id = aid
feed.content = GetString(reader, 0)
feed.sender_id = GetInt64(reader, 2)
feed.send_time = GetInt64(reader, 3) / 1000
feed.source = files
string = GetString(reader, 4)
if string is not '':
js = json.loads(string)
for a in js:
feed.comments += '{},'.format(a.get('commentId'))
fcm = model_im.FeedComment()
fcm.comment_id = a.get('commentId')
fcm.content = a.get('commentContent').get('text')
fcm.sender_id = a.get('senderId')
fcm.sender_name = a.get('commenter')
fcm.create_time = a.get('createdAt') / 1000
fcm.source = files
self.im.db_insert_table_feed_comment(fcm)
self.im.db_insert_table_feed(feed)
cmd.Dispose()
conn.Close()
self.im.db_commit()
files = self.cache + '/{}_chat.db'.format(aid)
if not os.path.exists(files):
self.m_print('no chat sqlite file, parse exists!')
return
conn = sql.SQLiteConnection(
'Data Source = {}; ReadOnly = True'.format(files))
conn.Open()
cmd = sql.SQLiteCommand(conn)
cmd.CommandText('''
select cid, title , createAt, ownerId, groupIcon from tbconversation where type = 2
''')
reader = cmd.ExecuteReader()
while reader.Read():
g = model_im.Chatroom()
g.account_id = aid
g.chatroom_id = GetString(reader, 0)
g.name = GetString(reader, 1)
g.owner_id = GetInt64(reader, 3)
g.create_time = GetInt64(reader, 2) / 1000
g.source = files
self.im.db_insert_table_chatroom(g)
cmd.Dispose()
cmd.CommandText = '''
select tbl_name from sqlite_master where type = 'table' and tbl_name like 'tbmsg_%'
'''
reader = cmd.ExecuteReader()
tbl_list = list()
while reader.Read():
tbl_list.append(GetString(reade, 0))
cmd.Dispose()
for tbl in tbl_list:
cmd.CommandText = '''
select cid, mid, senderId, createdAt, contentType, content from {}
'''.format(tbl)
reader = cmd.ExecuteReader()
while reader.Read():
msg = model_im.Message()
msg.account_id = aid
msg.msg_id = GetInt64(reader, 1)
msg.talker_id = GetString(reader, 0)
# talker_name?
msg.sender_id = GetInt64(reader, 2)
msg.is_sender = 1 if msg.sender_id == aid else 0
msg.send_time = GetInt64(reader, 3) / 1000
tp = GetInt64(reader, 4)
string = GetString(reader, 5)
if tp == 1:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
js = json.loads(string)
msg.content = js.get('txt')
elif tp == 2:
msg.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
js = json.loads(string)
msg.content = js.get('url')
elif tp == 501 or tp == 503 or tp == 502:
msg.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT
msg.content = js.get('ext').get('f_name')
elif tp == 901: # redpakcet
#msg.type = model_im.message_content_type_
# pass
js = json.loads(string)
amount = js.get('ext').get('amount')
title = js.get('ext').get('congrats')
size = js.get('ext').get('size')
elif tp == 203: # picture...
msg.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT
js = json.loads(string)
msg.content = js.get('f_name')
elif tp == 1200 or tp == 1203:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
js = json.loads(string)
msg.content = js.get('markdown')
elif tp == 300:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
js = json.loads(string)
ext = js.get('multi')[0].get('ext')
js = json.loads(ext)
msg.content = js.get('b_tl')
elif tp == 103:
msg.type = model_im.MESSAGE_CONTENT_TYPE_VIDEO
js = json.loads(string)
msg.content = js.get('url')
# msg.media_path = msg.content # ???
elif tp == 1101:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
msg.content = string
elif tp == 3:
msg.type = model_im.MESSAGE_CONTENT_TYPE_VOICE
msg.content = json.loads(string).get('url')
elif tp == 1600:
msg.type = model_im.MESSAGE_CONTENT_TYPE_SYSTEM
msg.content = json.loads(string).get('dingContent')
else:
msg.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
msg.content = string
self.im.db_insert_table_message(msg)
self.im.db_commit()
cmd.Dispose()
conn.Close()
def parse_Feed(self, plist_node=None):
''' 5A249183-668C-4CC0-B983-C0A7EA2E657F\
Library\Caches\PrivateStore\
P_u423af962f1456db6cba8465cf82bb91b\jp.naver.myhome.MBDataResults\
myhomelist
timeline
'''
'''
self.account_id = None # 账号ID[TEXT]
self.sender_id = None # 发布者ID[TEXT]
self.type = None # 动态类型[INT]
self.content = None # 动态内容[TEXT]
self.media_path = None # 媒体文件地址[TEXT]
self.urls = None # 链接地址[json TEXT] json string ['url1', 'url2'...]
# 预览地址[json TEXT] json string ['url1', 'url2'...]
self.preview_urls = None
self.attachment_title = None # 附件标题[TEXT]
self.attachment_link = None # 附件链接[TEXT]
self.attachment_desc = None # 附件描述[TEXT]
self.send_time = None # 发布时间[INT]
self.likes = None # 赞[TEXT] 逗号分隔like_id 例如：like_id, like_id, like_id, ...
self.likecount = None # 赞数量[INT]
self.rtcount = None # 转发数量[INT]
# 评论[TEXT] 逗号分隔comment_id 例如：comment_id,comment_id,comment_id,...
self.comments = None
self.commentcount = None # 评论数量[INT]
self.device = None # 设备名称[TEXT]
self.location = None # 地址ID[TEXT]
'''
def _print_plist(n, ind=''):
print ind + 'key: {}'.format(n.Key)
if n.Values:
for chch in n.Values:
_print_plist(chch, ind=ind + '|----')
else:
print ind + 'val:{}'.format(n.Value)
bplist = BPReader.GetTree(plist_node.Data) if plist_node else None
if not bplist:
return
for feed_node in bplist:
#_print_plist(feed_node)
feed = model_im.Feed()
feed.account_id = self.cur_account_id # 账号ID[TEXT]
feed.sender_id = feed_node['fromUser'].Value # 发布者ID[TEXT]
feed.content = feed_node['contents']['text'].Value # 动态内容[TEXT]
# 图片 视频
media_node = feed_node['contents']['media']
try:
if media_node:
for key in ('photos', 'videos'): # extra 'medias'
if not media_node[key]:
continue
path_res = []
for each_media_node in media_node[key]:
_path = each_media_node['sourceURL'].Value
if _path:
path_res.append(_path)
if key == 'photos':
feed.image_path = ','.join(x for x in path_res)
elif key == 'videos':
feed.video_path = ','.join(x for x in path_res)
except:
exc()
# _print_plist(feed_node)
# 链接
additional_node = feed_node['contents']['additionalContents']
if additional_node and additional_node['url']:
feed.url = additional_node['url']['targetUrl'].Value
if feed.url and additional_node['title']:
feed.url_title = additional_node['title'].Value
# location
loc_node = feed_node['contents']['textLocation']['location']
if loc_node and loc_node['latitude'] and loc_node['longitude']:
location = model_im.Location()
location.address = loc_node['name'].Value
location.timestamp = feed_node['postInfo']['createdTime'].Value
location.latitude = loc_node['latitude'].Value
location.longitude = loc_node['longitude'].Value
try:
self.csm.db_insert_table_location(location)
except:
exc()
feed.location = location.location_id
feed.send_time = feed_node['postInfo'][
'createdTime'].Value # 发布时间[INT]
feed.likecount = feed_node['postInfo'][
'likeCount'].Value # 赞数量[INT]
feed.rtcount = feed_node['postInfo']['sharedCount']['toPost'].Value \
+ feed_node['postInfo']['sharedCount']['toTalk'].Value # 转发数量[INT]
feed.commentcount = feed_node['postInfo'][
'commentCount'].Value # 评论数量[INT]
feed.source = plist_node.AbsolutePath
try:
self.csm.db_insert_table_feed(feed)
except:
exc()
self.csm.db_commit()
