def main(input=None, debug_mode=False, quiet_mode=False):
# TODO: print program information.
if not input:
argument_parser = argparse.ArgumentParser(description='Android Basic Apps Parser.')
argument_parser.add_argument('-i', '--input_path', required=True, action='store',
help='Path to input file/directory')
argument_parser.add_argument('-o', '--output_path', required=True, action='store', help='Output path')
argument_parser.add_argument('-d', '--debug', default=False, action='store', help='set debug mode')
argument_parser.add_argument('-q', '--quiet', default=False, action='store', help='set quiet mode')
args = argument_parser.parse_args()
input_path = args.input_path
output_path = os.path.abspath(args.output_path)
_debug_mode = args.debug
_quiet_mode = args.quiet
else:
input_path = input
output_path = input
_debug_mode = debug_mode
_quiet_mode = quiet_mode
if len(input_path) == 0:
print('invalid input file or directory.')
return
if len(output_path) == 0:
print('invalid output file or directory.')
return
# set output directory and logfile
local_date_time = datetime.datetime.now()
_result_path = '{0:s}AB2A_Results-{1:04d}{2:02d}{3:02d}T{4:02d}{5:02d}{6:02d}'.format(
output_path + os.sep, local_date_time.year, local_date_time.month,
local_date_time.day, local_date_time.hour, local_date_time.minute,
local_date_time.second)
_log_file = '{0:s}{1:s}-{2:04d}{3:02d}{4:02d}T{5:02d}{6:02d}{7:02d}.log.gz'.format(
_result_path, os.sep + 'AB2A', local_date_time.year, local_date_time.month,
local_date_time.day, local_date_time.hour, local_date_time.minute,
local_date_time.second)
loggers.ConfigureLogging(debug_output=_debug_mode, filename=_log_file,
quiet_mode=_quiet_mode)
results = []
for key, val in SUPPORTED_BASIC_APPS.items():
search_results = search(input_path, val[0])
if not search_results:
if not os.path.exists(_result_path):
os.mkdir(_result_path)
logger.info(f'No results: {key}:{val[0]}')
else:
result = process(search_results, val[1], _result_path)
if result:
for ret in result:
results.append(ret)
return results
def parse_sim_info(target_files, result_path):
"""Parse SIM information databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse SIM information databases.')
results = []
for file in target_files:
if str(file).endswith('telephony.db'):
database = sqlite3.connect(str(file))
database.row_factory = sqlite3.Row
result = _parse_sim_info(database, result_path)
if result:
results.append(result)
elif str(file).endswith('SimCard.dat'):
result = _parse_simcard_dat(str(file), result_path)
if result:
results.append(result)
return results
def parse_system_info(target_files, result_path):
"""Parse system info databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse system information databases.')
results = []
for file in target_files:
if str(file).endswith('settings_secure.xml'):
result = _parse_system_info(str(file), result_path)
if result:
results.append(result)
return results
def parse_wifi(target_files, result_path):
"""Parse Wi-Fi profile xml file.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse Wi-Fi profile XML file.')
results = []
for file in target_files:
if str(file).endswith('WifiConfigStore.xml'):
result = _parse_wifi(str(file), result_path)
if result:
results.append(result)
return results
def parse_usagestats(target_files, result_path):
"""Parse android usagestats.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse usagestats')
results = []
for file in target_files:
if os.path.dirname(str(file)).endswith('usagestats'):
uid = str(file).split(os.sep)[-1]
result = _parse_usagestats(str(file), uid, result_path)
if result:
results.append(result)
return results
def parse_file_cache(target_files, result_path):
"""Parse file cache databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse file cache databases.')
results = []
for file in target_files:
if str(file).endswith('FileCache.db'):
database = sqlite3.connect(str(file))
database.row_factory = sqlite3.Row
result = _parse_file_cache(database, result_path)
if result:
results.append(result)
return results
def parse_user_dict(target_files, result_path):
"""Parse user_dict databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse User Dictionary databases.')
results = []
for file in target_files:
if str(file).endswith('user_dict.db'):
database = sqlite3.connect(str(file))
database.row_factory = sqlite3.Row
result = _parse_user_dict(database, result_path)
if result:
results.append(result)
return results
def parse_accounts_de(target_files, result_path):
"""Parse accounts_de databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse accounts_de databases.')
results = []
for file in target_files:
if str(file).endswith('accounts_de.db'):
uid = str(file).split(os.sep)[-2]
database = sqlite3.connect(str(file))
database.row_factory = sqlite3.Row
result = _parse_accounts_de(database, uid, result_path)
if result:
results.append(result)
return results
def parse_smsmms(target_files, result_path):
"""Parse SMS and MMS databases.
Args:
target_files (list): target files.
result_path (str): result path.
"""
logger.info('Parse SMS and MMS databases.')
results = []
for file in target_files:
if str(file).endswith('mmssms.db'):
database = sqlite3.connect(str(file))
database.row_factory = sqlite3.Row
sms_data = _parse_sms(database, result_path)
if sms_data:
results.append(sms_data)
mms_data = _parse_mms(database, result_path)
if mms_data:
results.append(mms_data)
return results
def _parse_mms(database, result_path):
"""Parse MMS messages.
Args:
database (SQLite3): target SQLite3 database.
result_path (str): result path.
"""
parent_path = pathlib.Path(result_path)
parent_path = parent_path.parent
cursor = database.cursor()
cursor.execute(mms_query)
results = cursor.fetchall()
num_of_results = len(results)
data = {}
data['title'] = 'mms'
header = ('mms_id', 'thread_id', 'date', 'date_sent', 'read', 'from', 'to',
'cc', 'bcc', 'body')
data['number_of_data_headers'] = len(header)
data['number_of_data'] = num_of_results
data['data_header'] = header
data_list = []
if num_of_results > 0:
for row in results:
if row['date_sent'] != 0:
msg = MmsMessage(
row['mms_id'], row['thread_id'],
datetime.datetime.fromtimestamp(
row['date'], datetime.timezone.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%fZ'),
datetime.datetime.fromtimestamp(
float(row['date_sent']) / 1000,
datetime.timezone.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%fZ'), row['read'], row['FROM'],
row['TO'], row['CC'], row['BCC'], row['msg_box'],
row['part_id'], row['seq'], row['ct'], row['cl'],
row['_data'], row['text'])
else:
msg = MmsMessage(
row['mms_id'], row['thread_id'],
datetime.datetime.fromtimestamp(
row['date'], datetime.timezone.utc).strftime(
'%Y-%m-%dT%H:%M:%S.%fZ'), '', row['read'],
row['FROM'], row['TO'], row['CC'], row['BCC'],
row['msg_box'], row['part_id'], row['seq'], row['ct'],
row['cl'], row['_data'], row['text'])
if row['_data'] == None:
msg.body = row['text']
else:
#TODO: attachment is existed!
result = _search(
parent_path,
'**' + os.sep + os.path.basename(row['_data']))
if result:
msg.filename = str(result[0])
msg.body = msg.filename
else:
logger.info('Attachment file is not found!'.format(
row['_data']))
temp = (msg.mms_id, msg.thread_id, msg.date, msg.date_sent,
msg.read, msg.From, msg.to, msg.cc, msg.bcc, msg.body)
data_list.append(temp)
data['data'] = data_list
return data
def _parse_usagestats(directory, uid, result_path):
"""Parse usagestats from /system/usagestats.
Args:
directory (str): target direcotry path.
uid (str): user id.
result_path (str): result path.
"""
data = {}
data['title'] = 'usagestats' + f'_{uid}'
header = ('usage_type', 'last_time_active', 'time_active_in_msecs', 'time_active_in_secs',
'last_time_service_used', 'last_time_visible', 'total_time_visible', 'app_launch_count',
'package', 'types', 'class', 'source', 'all_attributes')
data['number_of_data_headers'] = len(header)
data['data_header'] = header
data_list = []
for file in glob.iglob(os.path.join(directory, '**'), recursive=True):
if os.path.isfile(file):
source = None
filename = os.path.basename(file)
if os.path.dirname(file).endswith('daily'):
source = 'daily'
elif os.path.dirname(file).endswith('weekly'):
source = 'weekly'
elif os.path.dirname(file).endswith('monthly'):
source = 'monthly'
elif os.path.dirname(file).endswith('yearly'):
source = 'yearly'
try:
filename_int = int(filename)
except:
logger.error('Invalid File Name: {0:s}'.format(filename))
try:
tree = xml.etree.ElementTree.parse(file)
root = tree.getroot()
logger.info('processing: {0:s}'.format(file))
for elem in root:
if elem.tag == 'packages':
usagestat = UsageStats()
usagestat.source = source
usagestat.usage_type = elem.tag
for subelem in elem:
usagestat.all_attributes = json.dumps(subelem.attrib)
last_time_active = int(subelem.attrib['lastTimeActive'])
if last_time_active < 0:
usagestat.last_time_active = abs(last_time_active)
else:
usagestat.last_time_active = filename_int + last_time_active
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
usagestat.package = subelem.attrib['package']
usagestat.time_active_in_msecs = subelem.attrib['timeActive']
usagestat.time_active_in_secs = int(usagestat.time_active_in_msecs) / 1000
usagestat.app_launch_count = subelem.attrib.get('appLaunchCount', None)
data_list.append(
(usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used,
usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
elif elem.tag == 'configurations':
usagestat = UsageStats()
usagestat.source = source
usagestat.usage_type = elem.tag
for subelem in elem:
usagestat.all_attributes = json.dumps(subelem.attrib)
last_time_active = int(subelem.attrib['lastTimeActive'])
if last_time_active < 0:
usagestat.last_time_active = abs(last_time_active)
else:
usagestat.last_time_active = filename_int + last_time_active
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
usagestat.time_active_in_msecs = subelem.attrib['timeActive']
usagestat.time_active_in_secs = int(usagestat.time_active_in_msecs) / 1000
data_list.append(
(usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used,
usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
elif elem.tag == 'event-log':
usagestat = UsageStats()
usagestat.source = source
usagestat.usage_type = elem.tag
for subelem in elem:
usagestat.all_attributes = json.dumps(subelem.attrib)
time = int(subelem.attrib['time'])
if time < 0:
usagestat.last_time_active = abs(time)
else:
usagestat.last_time_active = filename_int + time
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
usagestat.package = subelem.attrib['package']
usagestat.types = str(EventType(int(subelem.attrib['type'])))
usagestat.cls = subelem.attrib.get('class', None)
data_list.append((usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used, usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
except xml.etree.ElementTree.ParseError:
# Perhaps an Android Q protobuf file
try:
stats = _ReadUsageStatsPbFile(file)
except:
logger.error('Parse Error: Non XML file and Non Protobuf file: {0:s}'.format(file))
continue
if stats:
for stat in stats.packages:
usagestat = UsageStats()
usagestat.source = source
usagestat.usage_type = 'packages'
if stat.HasField('last_time_active_ms'):
last_time_active = stat.last_time_active_ms
if last_time_active < 0:
usagestat.last_time_active = abs(last_time_active)
else:
usagestat.last_time_active = filename_int + last_time_active
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
if stat.HasField('total_time_active_ms'):
usagestat.time_active_in_msecs = abs(stat.total_time_active_ms)
usagestat.package = stats.stringpool.strings[usagestat.package_index - 1]
if stat.HasField('app_launch_count'):
usagestat.app_launch_count = abs(stat.app_launch_count)
data_list.append(
(usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used,
usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
for stat in stats.configurations:
usagestat = UsageStats()
usagestat.source = source
usagestat.usage_type = 'configurations'
if stat.HasField('last_time_active_ms'):
last_time_active = stat.last_time_active_ms
if last_time_active < 0:
usagestat.last_time_active = abs(last_time_active)
else:
usagestat.last_time_active = filename_int + last_time_active
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
if stat.HasField('total_time_active_ms'):
usagestat.time_active_in_msecs = abs(stat.total_time_active_ms)
usagestat.all_attributes = str(stat.config)
data_list.append(
(usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used,
usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
for stat in stats.event_log:
if stat.HasField('time_ms'):
last_time_active = stat.time_ms
if last_time_active < 0:
usagestat.last_time_active = abs(last_time_active)
else:
usagestat.last_time_active = filename_int + last_time_active
usagestat.last_time_active = datetime.datetime.fromtimestamp(usagestat.last_time_active / 1000,
datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
if stat.HasField('package_index'):
usagestat.package = stats.stringpool.strings[stat.package_index - 1]
if stat.HasField('package_index'):
usagestat.cls = stats.stringpool.strings[stat.class_index - 1]
if stat.HasField('type'):
usagestat.types = str(EventType(stat.type)) if stat.type <= 18 else str(stat.type)
data_list.append(
(usagestat.usage_type, usagestat.last_time_active, usagestat.time_active_in_msecs,
usagestat.time_active_in_secs, usagestat.last_time_service_used,
usagestat.last_time_visible,
usagestat.total_time_visible, usagestat.app_launch_count, usagestat.package,
usagestat.types, usagestat.cls, usagestat.source, usagestat.all_attributes))
continue
data['number_of_data'] = len(data_list)
data['data'] = data_list
return data