def get_photo_from_id(self, photo_id):
hash_str = unity_c37r.md5(photo_id)
f_name = os.path.join(self.root.PathWithMountPoint, 'Caches/%s' %hash_str)
if os.path.exists(f_name):
return '{}/Caches/{}'.format(self.root.AbsolutePath, hash_str)
else:
return ""
def __init__(self, node, extract_deleted, extract_source):
self.node = node
self.extract_deleted = extract_deleted
self.extract_source = extract_source
self.cache = ds.OpenCachePath('xianyu')
self.im = model_im.IM()
self.hash = unity_c37r.md5(node.PathWithMountPoint)
self.im.db_create(self.cache + '/{}'.format(self.hash))
self.account = None
def __init__(self, node, extract_deleted, extract_source):
self.cache = ds.OpenCachePath('baidunetdisk')
if not os.path.exists(self.cache):
os.mkdir(self.cache)
self.hash = unity_c37r.md5(node.PathWithMountPoint)
self.nd = model_nd.NetDisk(self.cache + '/{}'.format(self.hash), BDY_APP_VERSION)
self.need_parse = self.nd.need_parse
if not self.need_parse:
return
self.node = node
self.extract_deleted = extract_deleted
self.extract_source = extract_source
self.account_folders = list()
def __init__(self, node, extract_deleted, extract_source):
global MM_PARSER_VERSION
self.node = node
self.extract_deleted = extract_deleted
self.extract_source = extract_source
self.cache = ds.OpenCachePath('momo')
self.hash = unity_c37r.md5(node.PathWithMountPoint)
self.cache_db = os.path.join(self.cache, self.hash)
self.need_parse = False
self.im = model_im.IM()
if model_im.IM.need_parse(self.cache_db, MM_PARSER_VERSION):
self.need_parse = True
self.im.db_create(self.cache_db)
self.accounts = list()
self.f_dict = dict()
def __init__(self, root, container_root, extract_deleted, extract_source, tp = 0):
self.root = root
self.extract_deleted = extract_deleted
self.extract_source = extract_source
self.account = None
self.cache = ds.OpenCachePath('Tg')
self.hash_code = unity_c37r.md5(container_root.AbsolutePath)
self.need_parse = False
self.im = model_im.IM()
self.container = container_root
self.tp = tp
self.__find_account()
if self.im.need_parse(self.cache + '/{}.C37R'.format(self.hash_code), 1):
self.im.db_create(self.cache + '/{}.C37R'.format(self.hash_code))
self.need_parse = True
else:
self.need_parse = False
def __init__(self, root, extract_deleted, extract_source):
self.root_node = root
self.extract_deleted = extract_deleted
self.extract_source = extract_source
self.im = model_im.IM()
self.cache = ds.OpenCachePath('miliao')
self.hash = unity_c37r.md5(root.PathWithMountPoint)
self.need_parse = False
if not os.path.exists(self.cache):
os.mkdir(self.cache)
if self.im.need_parse(self.cache + "/{}".format(self.hash),
VERSION_APP_VALUE):
self.im.db_create(os.path.join(self.cache, self.hash))
self.need_parse = True
else:
self.need_parse = False
self.account = list()
self.current_login = None
def __init__(self, node, extract_source, extract_deleted, is_scripts=True):
self.node = node
self.extract_source = extract_source
self.extract_deleted = extract_deleted
self.account_list = list()
self.account_dbs = dict()
self.im = model_im.IM()
self.cache = ''
if is_scripts:
self.cache = ds.OpenCachePath('Twitter')
else:
self.cache = "D:/Cache"
self.hash = unity_c37r.md5(self.node.AbsolutePath)
self.cache_db = os.path.join(self.cache, self.hash)
self.need_parse = model_im.IM.need_parse(self.cache_db,
AND_TWR_VERSION)
if self.need_parse:
self.im.db_create(self.cache_db)
def __init__(self, node, extract_source, extract_deleted):
self.node = node
self.extract_source = extract_source
self.extract_deleted = extract_deleted
self.cache = ds.OpenCachePath('taobao')
self.hash = unity_c37r.md5(node.PathWithMountPoint.encode('utf-8'))
self.eb = model_eb.EB(self.cache + '/{}'.format(self.hash), TB_VERSION,
u'淘宝')
self.im = self.eb.im
self.need_parse = self.eb.need_parse
self.master_account = None # 从Documents/../TBSettingsUserInfo.json(xml) 中获取
if self.need_parse:
self.eb.db_create()
self.get_master_account()
self.account = list()
self.log = self.eb.log
self.message_dict = dict() # 放关于message对应的表
self.avfs_message_dict = dict()
self.log.set_level(1) # 开启双重print
def __init__(self, root, extract_deleted, extract_source):
self.node = root
self.es = extract_source
self.ed = extract_deleted
self.account = list()
self.friends = list()
self.messages = list()
self.blogs = list()
self.search = list()
self.need_parse = False
cache = ds.OpenCachePath('Twitter')
self.cache = cache
if not os.path.exists(cache):
os.mkdir(cache)
#self.mb = microblog_base.BlogBase(env)
self.im = model_im.IM()
self.hash = unity_c37r.md5(root.PathWithMountPoint)
if self.im.need_parse(cache + '/{}'.format(self.hash),
VERSION_APP_VALUE):
self.im.db_create(cache + '/{}'.format(self.hash))
self.need_parse = True
self.db_dict = dict()
def __parse_message(self, a_node, aid):
f_message_node = a_node.GetByPath('/{}_Message.sqlite'.format(aid))
db = unity_c37r.create_connection(f_message_node.PathWithMountPoint)
cmd = sql.SQLiteCommand(db)
cmd.CommandText = '''
select ZLOCAL_ID, ZBODY_TYPE, ZTIMESTAMP, ZBODY, ZEXT_ID, ZMSG_SENDER, ZMSG_TO, ZMSG_XML from ZMESSAGEV5OBJECT
'''
try:
reader = None
reader = cmd.ExecuteReader()
except: #因为此处会出现错误，故暂时在出异常时，把文件移到Cache里面（Md5)
if reader is not None:
reader.Close()
cmd.Dispose()
db.Close()
dst_file = self.cache + "/" + unity_c37r.md5(
f_message_node.PathWithMountPoint)
unity_c37r.mapping_file_with_copy(
f_message_node.PathWithMountPoint, dst_file)
db = unity_c37r.create_connection(dst_file)
cmd = sql.SQLiteCommand(db)
cmd.CommandText = '''
select ZLOCAL_ID, ZBODY_TYPE, ZTIMESTAMP, ZBODY, ZEXT_ID, ZMSG_SENDER, ZMSG_TO, ZMSG_XML from ZMESSAGEV5OBJECT
'''
reader = cmd.ExecuteReader() # 如果再出错误，这个案例就有问题了
while reader.Read():
if canceller.IsCancellationRequested:
self.im.db_close()
raise IOError("E")
m = model_im.Message()
m.talker_type = model_im.CHAT_TYPE_FRIEND
m.account_id = aid
target_id = unity_c37r.c_sharp_get_string(reader, 5)
m.is_sender = 1 if str(target_id) == aid else 0
m.deleted = 0
m.msg_id = unity_c37r.c_sharp_get_long(reader, 0)
m.send_time = int(
unity_c37r.c_sharp_get_real(reader, 2) * 1000) / 1000
m.source = f_message_node.AbsolutePath
m.sender_id = unity_c37r.c_sharp_get_string(reader, 5)
if m.is_sender == 1:
tid = unity_c37r.c_sharp_get_string(reader, 6)
m.talker_id = tid.split('@')[0]
else:
m.talker_id = unity_c37r.c_sharp_get_string(reader, 5)
#def __process_message(self, content, xml, aid, m_type, ext_id):
p1 = unity_c37r.c_sharp_get_string(reader, 3)
p2 = unity_c37r.c_sharp_get_string(reader, 7)
tp = unity_c37r.c_sharp_get_long(reader, 1)
p3 = unity_c37r.c_sharp_get_string(reader, 4)
s_msg = self.__process_message(p1, p2, aid, tp, p3)
m.content = s_msg.content
m.media_path = s_msg.media_path
m.type = s_msg.media_type
try:
self.im.db_insert_table_message(m)
except:
print('f****d!')
cmd.Dispose()
db.Close()
self.im.db_commit()
g_message_node = self.root_node.GetByPath(
'Documents/{}/account/{}_MUC.sqlite'.format(aid, aid))
if g_message_node is None:
reader = None
else:
db = unity_c37r.create_connection(
g_message_node.PathWithMountPoint)
cmd = sql.SQLiteCommand(db)
cmd.CommandText = '''
select ZBODY_TYPE, ZMSG_ID, ZSEND_TIME, ZBODY, ZSENDER_ID, ZMSG_TO, ZMSG_XML, ZEXT_ID from ZMUCMESSAGEOBJECT
'''
reader = cmd.ExecuteReader()
while reader is not None and reader.Read():
if canceller.IsCancellationRequested:
self.im.db_close()
raise IOError("E")
m = model_im.Message()
m.talker_type = model_im.CHAT_TYPE_GROUP
m.msg_id = unity_c37r.c_sharp_get_string(reader, 1)
m.account_id = aid
sender_id = unity_c37r.c_sharp_get_string(reader, 4)
m.sender_id = sender_id if sender_id is not 0 or unity_c37r.c_sharp_get_string(
reader, 4) != '' else -1000 # -1000 means system...
m.is_sender = 1 if str(m.sender_id) == aid else 0
m.send_time = unity_c37r.c_sharp_get_long(reader, 2) / 1000
m.source = g_message_node.AbsolutePath
m.deleted = 0
tp = unity_c37r.c_sharp_get_long(reader, 0)
xml = unity_c37r.c_sharp_get_string(reader, 6)
ext_id = unity_c37r.c_sharp_get_string(reader, 7)
if tp == 0 or tp == 3:
m.content = unity_c37r.c_sharp_get_string(reader, 3)
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT if m.sender_id != -1000 else model_im.MESSAGE_CONTENT_TYPE_SYSTEM
elif xml is not None and ext_id is None:
if tp == 1:
r = os.path.exists(
os.path.join(self.root_node.PathWithMountPoint,
'Documents/image/%s' % ext_id))
if r:
m.media_path = os.path.join(
self.root_node.AbsolutePath,
'Documents/image/%s' % ext_id)
m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
elif os.path.exists(
os.path.join(self.root_node.PathWithMountPoint,
'Documents/image/th_%s' % ext_id)):
m.media_path = os.path.join(
self.root_node.AbsolutePath,
'Documents/image/th_%s' % ext_id)
m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
else:
m.content = 'image message not cached'
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 6:
r = self.__handle_location(aid, ext_id)
if r == '':
m.content = 'location message not cached'
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
r = r.split(';')
m.location_name = r[0]
m.location_lat = r[1]
m.location_lng = r[2]
m.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION
else:
p1 = unity_c37r.c_sharp_get_string(reader, 3)
p2 = unity_c37r.c_sharp_get_string(reader, 6)
ext_id = unity_c37r.c_sharp_get_string(reader, 7)
s_msg = self.__process_message(p1, p2, aid, tp, ext_id)
m.media_path = s_msg.media_path
m.type = s_msg.media_type
self.im.db_insert_table_message(m)
if reader is not None:
cmd.Dispose()
db.Close()
self.im.db_commit()
def parse(self, ac):
#ac = TbAccount()
self.parse_avfs(ac)
db_node = self.node.GetByPath('Library/Caches/amps3_{}.db'.format(
ac.uid))
if db_node is None:
self.log.m_print('get db node failed, parse exists!')
return
#path = db_node.PathWithMountPoint
#update sqlite-checking.
path = unity_c37r.check_sqlite_maturity(db_node, self.cache)
conn = unity_c37r.create_connection(path)
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select ZDISPLAYNAME, ZHEADPIC, ZMOBILEPHONE, ZNICK from ZAMPUSER where ZTAOBAOID = {}
'''.format(ac.uid)
reader = cmd.ExecuteReader()
image_cache_path = os.path.join(
self.node.PathWithMountPoint,
'Library/Caches/YWDiskCache/ImageCache')
if reader.Read():
a = model_im.Account()
a.account_id = ac.uid
a.nickname = unity_c37r.c_sharp_get_string(reader, 3)
a.username = unity_c37r.c_sharp_get_string(reader, 0)
ac.tb_id = a.username
a.telephone = unity_c37r.c_sharp_get_string(reader, 2)
pic = unity_c37r.c_sharp_get_string(reader, 1)
hash_code = unity_c37r.md5(pic)
if os.path.exists(os.path.join(image_cache_path, hash_code)):
a.photo = 'Library/Cache/YWDiskCache/YWDiskCache/ImageCache/{}'.format(
hash_code)
else:
a.photo = pic
self.im.db_insert_table_account(a)
cmd.Dispose()
cmd.CommandText = '''
select ZDISPLAYNAME, ZHEADPIC, ZMOBILEPHONE, ZNICK, ZTAOBAOID, ZSIGNATURE from ZAMPUSER where ZTAOBAOID != {}
'''.format(ac.uid)
reader = cmd.ExecuteReader()
f_dict = dict()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('E')
f = model_im.Friend()
f.account_id = ac.uid # Fix error
try:
f.friend_id = int(unity_c37r.c_sharp_get_string(reader, 4))
except Exception as e:
self.log.m_err('error account id: %s' %
unity_c37r.c_sharp_get_string(reader, 4))
f.friend_id = random.randint(0, 0xffffffff) # 产生假ID
f.nickname = unity_c37r.c_sharp_get_string(reader, 3)
f.remark = unity_c37r.c_sharp_get_string(reader, 0)
f.telephone = unity_c37r.c_sharp_get_string(reader, 2)
f.signature = unity_c37r.c_sharp_get_string(reader, 5)
pic = unity_c37r.c_sharp_get_string(reader, 1)
hash_code = unity_c37r.md5(pic)
if os.path.exists(os.path.join(image_cache_path, hash_code)):
f.photo = 'Library/Cache/YWDiskCache/ImageCache/{}'.format(
hash_code)
else:
f.photo = pic
f_dict[f.remark] = f
cmd.Dispose()
conn.Close()
conn = None
cmd = None
# 用户行为分析 可以说很详细了
if ac.tb_id is None:
reader = None
else:
log_node = self.node.GetByPath(
'Library/Caches/StructuredLogs/cntaobao{}.db'.format(ac.tb_id))
if log_node is None:
reader = None
else:
#update sqlite connection
pth = unity_c37r.check_sqlite_maturity(log_node, self.cache)
conn = unity_c37r.create_connection(pth)
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select id, operation_id, record, logtime, result from Record
'''
reader = cmd.ExecuteReader()
while reader is not None and reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
raise IOError('E')
try:
logs = model_eb.EBLog()
m_str = unity_c37r.c_sharp_get_string(reader, 1)
m_sl = m_str.split('|')
sender = m_sl[1]
reciever = m_sl[2]
if reciever == 'wwLogin':
sender = re.search(r'\(null\)(.*)', sender,
re.I | re.M).group(1)
desc = '''{} try to login'''.format(sender)
logs.set_value_with_idx(logs.description, desc)
else:
sender = re.search('cnhhupan(.*)', sender,
re.I | re.M).group(1)
reciever = re.search('cnhhupan(.*)', reciever,
re.I | re.M).group(1)
desc = '''{} try to send message to {}'''.format(
sender, reciever)
logs.set_value_with_idx(logs.description, desc)
m_str = unity_c37r.c_sharp_get_string(reader, 2)
js = json.loads(m_str)
content = js.get('title')
logs.set_value_with_idx(logs.content, content)
log_time = unity_c37r.c_sharp_get_long(reader, 3) / 1000
logs.set_value_with_idx(logs.time, log_time)
result = unity_c37r.c_sharp_get_long(reader, 4)
logs.set_value_with_idx(logs.result, result)
self.eb.db_insert_table_log(logs.get_value())
except Exception as e:
traceback.print_exc()
self.log.m_print(e)
self.log.m_err('detect wrong string format: {}'.format(m_str))
if conn is not None:
cmd.Dispose()
conn.Close()
cmd = None
conn = None
#l = unity_c37r.search_for_certain(self.node, 'Library/Caches/YWDB/WXOPENIMSDKDB(.*)/message.db$')
u = self.message_dict.get(ac.tb_id)
if u is None:
self.log.m_print('no %s chat info!' % ac.tb_id)
return
conn = unity_c37r.create_connection(u)
cmd = sql.SQLiteCommand(conn)
cmd.CommandText = '''
select ZDISPLAYNAME, ZEMAIL, ZGENDER, ZTBNICK, ZPHONE_NUM from ZWWPERSON
'''
reader = cmd.ExecuteReader()
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
user_name = unity_c37r.c_sharp_get_string(reader, 0)
if f_dict.__contains__(user_name):
pass
else:
f = model_im.Friend()
f.account_id = ac.uid
f.nickname = unity_c37r.c_sharp_get_string(reader, 3)
f.remark = unity_c37r.c_sharp_get_string(reader, 0)
f.friend_id = random.randint(0, 0xffffffff) # 产生假ID
f_dict[f.remark] = f
for k in f_dict:
self.im.db_insert_table_friend(f_dict[k])
cmd.Dispose()
cmd.CommandText = '''
select ZMESSAGEID, ZTYPE, ZTIME, ZCONTENT, ZRECEIVERID, ZSENDERID from ZWWMESSAGE
'''
reader = cmd.ExecuteReader()
# 最好将talker_name之类的设置全面，因为淘宝的id管理比较混乱。
# 原则上淘宝账号、支付宝账号、闲鱼账号、天猫账号、阿里巴巴账号等通用，但是实际上他们在进行管理时
# 各自采用不同的ID管理方式。导致这边筛选很尴尬
idx = 0
while reader.Read():
if canceller.IsCancellationRequested:
cmd.Dispose()
conn.Close()
self.im.db_close()
raise IOError('e')
m = model_im.Message()
m.account_id = ac.uid
sender = unity_c37r.c_sharp_get_blob(reader, 5).decode(
'utf-8'
) #struct.unpack('i', unity_c37r.c_sharp_get_blob(reader, 5))
reciever = unity_c37r.c_sharp_get_blob(reader, 4).decode(
'utf-8'
) #struct.unpack('i', unity_c37r.c_sharp_get_blob(reader, 4))
try:
sender = re.search("cnhhupan(.*)", sender,
re.I | re.M).group(1)
reciever = re.search("cnhhupan(.*)", reciever,
re.I | re.M).group(1)
except:
try:
if sender.__contains__('alichn'):
sender = re.search('cnalichn(.*)', sender,
re.I | re.M).group(1)
else:
reciever = re.search('cnalichn(.*)', reciever,
re.I | re.M).group(1)
except:
self.log.m_print("sender:{} rec:{}".format(
sender, reciever))
if sender.__contains__(':'):
sender = sender.split(':')[0]
if reciever.__contains__(':'):
reciever = reciever.split(':')[0]
if sender == ac.tb_id:
m.is_sender = 1
if not f_dict.__contains__(reciever):
self.log.m_print("no such reciever friend:%s" % reciever)
continue
m.talker_id = f_dict[reciever].friend_id
m.sender_id = ac.uid
elif f_dict.__contains__(sender):
m.is_sender = 0
m.talker_id = f_dict[sender].friend_id
m.sender_id = f_dict[sender].friend_id
elif f_dict.__contains__(sender): #类似于hqxuelang:服务助手 这种有时候就不在表中
m.is_sender = 0
m.talker_id = f_dict[sender].friend_id
m.sender_id = f_dict[sender].friend_id
else:
self.log.m_print('find unchecked friend:%s' % sender)
f = self.create_fake_friend(ac, sender)
f_dict[sender] = f
m.talker_id = f_dict[sender].friend_id
m.sender_id = f_dict[sender].friend_id
m.msg_id = unity_c37r.c_sharp_get_long(reader, 0)
tp = unity_c37r.c_sharp_get_long(reader, 1)
tm = unity_c37r.c_sharp_try_get_time(reader, 2)
m.send_time = unity_c37r.format_mac_timestamp(tm)
m_str = unity_c37r.c_sharp_get_blob(reader, 3)
if m_str is not None:
m_str = m_str.decode('utf-8')
else:
self.log.m_print('find useless data, skipped!')
m_str = ''
if tp == 0 or tp == 5:
m.content = m_str
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 1:
m.content = m_str
m.media_path = m_str
m.type = model_im.MESSAGE_CONTENT_TYPE_IMAGE
elif tp == 2:
m.content = m_str
m.media_path = m_str
m.type = model_im.MESSAGE_CONTENT_TYPE_VOICE
elif tp == 8:
m.content = 'empty message'
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
elif tp == 65:
try:
js = json.loads(m_str)
# 这里有子类型号
# 实际上是属于千牛发送的自动排列的消息内容
sub_tp = js.get('template').get('id')
sub_string = js.get('template').get('data').get('text')
# f = open('D:/webs/{}.xml'.format(random.randint(0, 0xffffffff)), 'w+')
# if sub_string is not None:
# f.write(sub_string)
# f.close()
if sub_tp == 20002:
m.type = model_im.MESSAGE_CONTENT_TYPE_ATTACHMENT
deal = model_im.Deal()
trade = model_eb.EBDeal()
alter_string = js.get('header').get('degrade').get(
'alternative')
if alter_string.__contains__(':'):
deal.deal_id = alter_string.split(':')[1]
m.extra_id = deal.deal_id
deal.description = '''title:{}\ncontent:{}'''.format(
js.get('header').get('title'),
js.get('header').get('degrade').get('alternative'))
trade.set_value_with_idx(trade.desc, deal.description)
deal.type = model_im.DEAL_TYPE_RECEIPT # may fix it later
trade.set_value_with_idx(trade.deal_type,
model_eb.TRADE_PAY)
#trade.set_value_with_idx(trade.status, model_eb.EBDEAL)
self.im.db_insert_table_deal(deal)
self.eb.db_insert_table_deal(trade.get_value())
elif sub_tp == 20013: # 快速入口
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
label_list = js.get('template').get('data').get(
'alist')
content = ''
for l in label_list:
content += '%s\n' % l.get('label')
m.content = content
elif sub_tp is None:
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''title:{}\ncontent:{}'''.format(
js.get('header').get('title'),
js.get('degrade').get('alternative'))
except Exception as e:
self.log.m_err(
'detect wrong message content:{}\nidx:{}'.format(
m_str, idx))
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = u'解析失败,原始内容:' + m_str
elif tp == 112:
js = json.loads(m_str)
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = js.get('title')
elif tp == 113:
js = json.loads(m_str)
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''file message name:{}, size:{}'''.format(
js.get('nodeName'), js.get('nodeSize'))
elif tp == 241:
js = json.loads(m_str)
js = json.loads(js.get('datas'))
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''[product info]name:{}\nprice:{}\nsales:{}\npic:{}'''.format(
js.get('name'), js.get('priceAsString'),
js.get('salesCount'), js.get('picUrl'))
else:
self.log.m_print('detect unspport type:{}, index:{}'.format(
tp, idx))
self.log.m_print('raw string: %s' % m_str)
m.type = model_im.MESSAGE_CONTENT_TYPE_TEXT
m.content = '''[unsupport type message] raw string:%s''' % m_str
self.im.db_insert_table_message(m)
idx += 1
self.im.db_commit()
