def _parse_notes(self, node):
note_files = node.Search(r'Note_\d+.txt')
deleted_note_files = self.root.Search(r'NoteDeleted_\d+.txt')
note_existing = 0
note_deleted = 1
files = [(f, note_existing)
for f in note_files] + [(f, note_deleted)
for f in deleted_note_files]
for f, is_deleted in files:
data = self._open_file(f)
for n in data.get('entries', []):
try:
note = model_notes.Notes()
note.html_content = n.get('snippet', None)
note.deleted = is_deleted
note.source = f.AbsolutePath
note.fold_id = n.get('folderId', None)
note.id = n['id']
note.modified = TimeHelper.convert_timestamp(
n.get('modifyDate', None))
note.created = TimeHelper.convert_timestamp(
n.get('createDate', None))
self.nm.db_insert_table_notes(note)
except Exception as e:
print(e)
self.nm.db_commit()
def _generate_ticket_table(self, json_file, account_id):
if json_file is None:
return
tickets = self._open_json_file(json_file).get('presp', {}).get('pdata', {}).get('activityList', [])
for ticket_info in tickets:
try:
activity_info = ticket_info.get('activityInfo', {})
ticket = model_map.LocationJourney()
ticket.account_id = account_id
ticket.flightid = activity_info.get('tktNo', None)
start_time = '{} {}'.format(activity_info.get('deptDateTz', ''), activity_info.get('deptTimeTz', ''))
ticket.start_time = TimeHelper.str_to_ts(start_time, _format='%Y-%m-%d %H:%M')
ticket.depart = activity_info.get('deptCityCode', None)
ticket.depart_address = activity_info.get('deptCityName', None)
end_time = '{} {}'.format(activity_info.get('destDateTz', ''), activity_info.get('destTimeTz', ''))
ticket.end_time = TimeHelper.str_to_ts(end_time, _format='%Y-%m-%d %H:%M')
ticket.destination = activity_info.get('destCityCode', None)
ticket.destination_address = activity_info.get('destCityName', None)
ticket.purchase_price = activity_info.get('priceJointWithUnit', None)
ticket.ticket_status = activity_info.get('tktStatusDesc', None)
ticket.order_time = activity_info.get('createTime', None)
ticket.latest_mod_time = activity_info.get('modifyTime', None)
self.csm.db_insert_table_journey(ticket)
except Exception as e:
print e
self.csm.db_commit()
def _parse_photos(self, node):
gallery_files = node.Search(r'Gallery_user_galleries_\d+.txt')
for f in gallery_files:
data = self._open_file(f)
for g in data.get('galleries', []):
try:
file_name = g.get('fileName', None)
if not file_name:
continue
pic = next(iter(self.root.Search(file_name)), None)
if not pic:
continue
m = model_media.Media()
m.source = f.AbsolutePath
m.url = pic.AbsolutePath
m.id = g.get('id', None)
m.title = g.get('title', None)
m.type = 'image'
m.modify_date = TimeHelper.convert_timestamp(
g.get('dateModified', None))
m.add_date = TimeHelper.str_to_ts(
g.get('exifinfo', {}).get('dateTime', None),
_format="%Y-%m-%d %H:%M:%S")
m.size = g.get('size', None)
m.height = g.get('exifinfo', {}).get('imageLength', None)
m.width = g.get('exifinfo', {}).get('imageWidth', None)
self.mm.db_insert_table_media(m)
except Exception as e:
print(e)
self.mm.db_commit()
def _main(self):
"""解析的逻辑主函数"""
db = self.root
IOSWords.connect(db)
models = []
for word in IOSWords.objects.all:
d = UserDictionary()
d.CreatetTime = TimeHelper.convert_timestamp_for_c_sharp(
TimeHelper.convert_ts_for_ios(int(word.ts)))
d.Phrase = word.words
d.ShortCut = word.key
models.append(d)
return models
def _generate_message_table(self, node, account_id):
sender_id = node
source = node.AbsolutePath
messages = self._open_json_file(node).get('messages', [])
for m in messages:
message = model_im.Message()
message.source = source
message.msg_id = m.get("clientmessageid", None)
message.account_id = account_id
message.talker_id = m.get("conversationid")
message.sender_id = message.sender_name = sender_id = message.sender_name = sender_id = m.get(
'from').split('/')[-1]
message.is_sender = 1
content = self.convert_message_content(m.get("content", None),
message)
message.content = content
send_time = TimeHelper.str_to_ts(m.get('originalarrivaltime',
None)[:-5].replace(
'T', " "),
_format="%Y-%m-%d %H:%M:%S")
message.send_time = send_time
message.type = self.convert_message_content_type(
m.get("messagetype", None))
message.talker_type = model_im.CHAT_TYPE_GROUP if "@" in message.talker_id else model_im.CHAT_TYPE_FRIEND
# TODO media path 无法添加，案例数据没有
self.model_im_col.db_insert_table_message(message)
self.model_im_col.db_commit()
def _insert_item_to_db(self, obj):
f = model_nd.NDFileList()
f.set_value_with_idx(f.account, self.account_id)
f.set_value_with_idx(f.file_name, obj.get('name', None))
f.set_value_with_idx(f.file_size, obj.get('Size', None))
f.set_value_with_idx(f.update_time, TimeHelper.str_to_ts(obj.get('FileModifyedTime', None),
_format="%Y-%m-%dT%H:%M:%S"))
f.set_value_with_idx(f.server_path, obj.get('Path', None))
self.model_nd.db_insert_filelist(f.get_values())
def _generate_deal_table(self):
deal_db = self._search_file('MyJourneyDB.sqlite$')
if not deal_db:
return
FlightTicket.connect(deal_db)
account_id = self.master_account.account_id
for ticket in FlightTicket.objects.all:
try:
deal = model_eb.EBDeal()
deal.set_value_with_idx(deal.account_id, account_id)
deal.set_value_with_idx(deal.deleted, ticket.deleted)
deal.set_value_with_idx(deal.source_file, ticket.source_path)
deal.set_value_with_idx(
deal.begin_time,
TimeHelper.str_to_ts(
"{} {}".format(ticket.dept_flight_date,
ticket.departure_time),
"%Y-%m-%d %H:%M"))
deal.set_value_with_idx(
deal.end_time,
TimeHelper.str_to_ts(
"{} {}".format(ticket.dest_flight_date,
ticket.destination_time),
"%Y-%m-%d %H:%M"))
deal.set_value_with_idx(
deal.content,
self._process_ticket_content(
ticket.departure_name,
ticket.destination_name,
"{} {}".format(ticket.dept_flight_date,
ticket.departure_time),
"{} {}".format(ticket.dest_flight_date,
ticket.destination_time),
))
deal.set_value_with_idx(
deal.status, self._process_ticket_status(ticket.status))
deal.set_value_with_idx(deal.deal_type,
model_eb.EBDEAL_TYPE_REC)
deal.set_value_with_idx(deal.target, ticket.ticket_id)
self.model_eb_col.db_insert_table_deal(deal.get_value())
except Exception as e:
self.logger.error()
self.model_eb_col.db_commit()
def _parse_sms(self, node):
sms_files = node.Search(r'Sms_\d+.txt')
for file_ in sms_files:
data = self._open_file(file_)
phone_calls = data.get('phonecall_view', {}).get('entries', [])
sms = data.get('entries', [])
for pc in phone_calls:
try:
record = model_callrecord.Records()
record.source = file_.AbsolutePath
record.duration = pc.get('duration', None)
record.phone_number = pc.get('number', None)
record.id = pc.get('id', None)
record.date = TimeHelper.convert_timestamp(
pc.get('date', None))
self.crm.db_insert_table_call_records(record)
except Exception as e:
print(e)
for s in sms:
try:
s = s['entry']
message = model_sms.SMS()
message.source = file_.AbsolutePath
message.body = s.get('snippet', None)
message.send_time = TimeHelper.convert_timestamp(
s.get('localTime', None))
message.read_status = 1 if s['unread'] == 0 else 0
message.is_sender = 1 if s['folder'] == 1 else 0
message._id = s.get('id', None)
recipients = s.get('recipients', None)
message.sender_phonenumber, message.recv_phonenumber = (self.owner_phone, recipients) \
if message.is_sender else (recipients, self.owner_phone)
self.sms.db_insert_table_sms(message)
except Exception as e:
print(e)
self.crm.db_commit()
self.sms.db_commit()
def _generate_search_table(self):
"""添加search记录"""
db_col = self.soul_app_col
table = db_col.get_table(
"search_record_post", {
"_id": [FieldType.Text, FieldConstraints.NotNull],
"ts": [FieldType.Int, FieldConstraints.NotNull],
})
for record in db_col.read_records(table):
try:
s = model_im.Search()
s.key = record['_id'].Value
s.create_time = TimeHelper.str_to_ts(
record['ts'].Value, _format="%Y-%m-%d %H:%M:%S")
s.deleted = 1 if record.IsDeleted else 0
s.source = db_col.db_path
self.model_im_col.db_insert_table_search(s)
except Exception as e:
self.logger.error()
self.model_im_col.db_commit()
def _generate_account_table(self):
"""
创建account table
"""
msg_db_col = self.message_col
account_id = os.path.basename(msg_db_col.db_path).split(".")[0]
db_col = self.soul_app_col
table = db_col.get_table(
"user", {
"_id": [FieldType.Int, FieldConstraints.NotNull],
"user": [FieldType.Text, FieldConstraints.NotNull],
})
for rec in db_col.read_records(table):
try:
if str(rec['_id'].Value) == account_id:
account_info = TaoUtils.json_loads(rec['user'].Value)
if not account_info:
continue
account = model_im.Account()
account.account_id = rec['_id'].Value
account.source = db_col.db_path
account.signature = account.username = account.nickname = account_info.get(
"signature", None)
account.gender = model_im.GENDER_FEMALE if account_info.get("gender", None) == "FEMALE" \
else model_im.GENDER_MALE
account.birthday = TimeHelper.convert_timestamp(
account_info.get("birthday", None))
account.email = account_info.get('bindMail', None)
account.country = "China" if account_info.get(
'area', None) else None
account.deleted = 1 if rec.IsDeleted else 0
else:
account = model_im.Account()
account.account_id = account.username = account.nickname = account_id
account.source = db_col.db_path
self.model_im_col.db_insert_table_account(account)
except Exception as e:
self.logger.error()
self.model_im_col.db_commit()
def _generate_account_table(self):
"""
创建account table
"""
account_file = self._search_account_info_file()
if not account_file:
return
file_info = PlistHelper.ReadPlist(account_file)
if not file_info:
return
for account_id in self._account_list:
try:
if not account_file:
return
account = model_im.Account()
account.account_id = account.username = account.nickname = account_id
account.email = file_info.Get('USER_MAIL' + account_id)
birth_day = file_info.Get('currentUserBrithday' + account_id)
if birth_day:
account.birthday = TimeHelper.str_to_ts(
birth_day.ToString())
if str(file_info['userInformation'].Get(
"chatUserID")) == account_id:
account.password = file_info['userInformation'].Get(
"loginUserPassWord")
account.telephone = file_info['userInformation'].Get(
"loginUserPhone")
account.signature = file_info["signatureText"]
account.username = file_info["userChatUsername"]
account.gender = model_im.GENDER_FEMALE if file_info[
"gender"] == "女" else model_im.GENDER_MALE
account.nickname = account.signature
# account.photo = account_info.get("account_info", {}).get("photo_url", None)
account.source = account_file.PathWithMountPoint
self.model_im_col.db_insert_table_account(account)
except Exception as e:
self.logger.error()
self.model_im_col.db_commit()
def convert_message_content(self, content, msg_obj):
content = content.strip()
if not (content.startswith("<") and content.endswith(">")):
return content
try:
hp = PaHtmlParser()
hp.feed(content)
hp.close()
tag_name = hp.first_dom.tag_name
if tag_name == "deletemember":
executor = hp.root['deletemember']['initiator'].data
target = hp.root['deletemember']['target'].data
return self.assemble_message_delete_member(executor, target)
elif tag_name == "addmember":
executor = hp.root['addmember']['initiator'].data
target = hp.root['addmember']['target'].data
return self.assemble_message_add_member(executor, target)
elif tag_name == "ss":
nodes = hp.root.get_all("ss")
return self.assemble_message_emoji(nodes)
elif tag_name == "uriobject":
if hp.root['uriobject'].get("swift", None):
encoded_info = hp.root['uriobject']['swift'].property[
'b64']
decoded_info = TaoUtils.decode_base64(encoded_info)
attachment = json.loads(decoded_info)["attachments"][0]
url = attachment["content"]['images'][0]['url']
return url
return hp.root['uriobject']['originalname'].property['v']
elif tag_name == 'partlist':
return self.assemble_message_call(hp.root, msg_obj.account_id)
elif tag_name == 'topicupdate':
return self.assemble_message_rename_group(hp.root)
elif tag_name == 'location':
address = hp.root['location'].property['address']
latitude = float(
hp.root['location'].property['latitude']) / 1000000
longitude = float(
hp.root['location'].property['longitude']) / 1000000
ts = TimeHelper.convert_timestamp(
hp.root['location'].property.get("timestamp", None))
location = model_im.Location()
location.account_id = self.using_account.account_id
location.address = address
location.latitude = latitude
location.longitude = longitude
location.timestamp = ts
self.model_im_col.db_insert_table_location(location)
msg_obj.location_id = location.location_id
msg_obj.type = model_im.MESSAGE_CONTENT_TYPE_LOCATION
return None
elif tag_name == 'historydisclosedupdate':
return self.assemble_message_history_closure(hp.root)
elif tag_name == 'joiningenabledupdate':
return self.assemble_message_close_add_memeber(hp.root)
elif tag_name == "sms":
return hp.root['sms']['defaults']['content'].data
elif tag_name == "pictureupdate":
return self.assemble_message_change_pic_bak(hp.root)
else:
return content
except Exception as e:
return content