def get_custom_app_name(preprocess_db_path):
if not os.path.exists(APP_PACKAGE_CONF_PATH):
logger.error('Not exist the config (\"%s\").' %
APP_PACKAGE_CONF_PATH)
return False
try:
f = open(APP_PACKAGE_CONF_PATH, 'r')
except Exception as e:
logger.error("Fail to open file [%s]" % APP_PACKAGE_CONF_PATH)
return False
lines = f.readlines()
for line in lines:
if not line: break
line = line.strip()
line = line.replace(" ", "")
line = line.replace("\t", "")
package_name = line.split(",")[0]
app_name = line.split(",")[1]
query = 'SELECT app_name FROM package_info WHERE package_name = "%s"' % package_name
ret = SQLite3.execute_fetch_query(query, preprocess_db_path)
if ret != None:
if ret[0] == "":
query = 'UPDATE package_info set app_name = "%s" WHERE package_name = "%s"' % (
app_name, package_name)
SQLite3.execute_commit_query(query, preprocess_db_path)
f.close()
def get_package_permission(package, preprocess_db_path):
cnt_perms = 0
perms_value = ""
for perms in package.iter('perms'):
for item in perms.iter('item'):
name = item.get('name')
perms_id = SystemLog.get_permission_id(name,
preprocess_db_path)
if perms_id != 0:
if perms_value == "":
perms_value = str(perms_id)
else:
perms_value += ","
perms_value += str(perms_id)
else:
query = 'INSERT INTO permission_info(permission_name) VALUES("%s")' % (
name)
SQLite3.execute_commit_query(query, preprocess_db_path)
perms_id = SystemLog.get_permission_id(
name, preprocess_db_path)
if perms_value == "":
perms_value = str(perms_id)
else:
perms_value = perms_value + "," + str(perms_id)
cnt_perms += 1
return cnt_perms, perms_value
def insert_extracted_files_info_to_resultdb(format_name, file_info,
preprocess_db_path,
extracted_file_path):
inode = file_info[0]
id_package = file_info[1]
parent_path = file_info[2]
name = file_info[3]
size = file_info[4]
ctime = file_info[5]
crtime = file_info[6]
atime = file_info[7]
mtime = file_info[8]
if format_name.upper() == "SQLITEDB":
query = 'INSERT INTO sqlitedb_info(inode, id_package, parent_path, name, size, ctime, crtime, atime, mtime, extracted_path) VALUES(%d, %d, "%s", "%s", %d, %d, %d, %d, %d, "%s")' % (
int(inode), int(id_package), parent_path, name, int(size),
int(ctime), int(crtime), int(atime), int(mtime),
extracted_file_path)
elif format_name.upper() == "APK":
query = 'INSERT INTO apk_file_info(inode, id_package, parent_path, name, size, ctime, crtime, atime, mtime, extracted_path) VALUES(%d, %d, "%s", "%s", %d, %d, %d, %d, %d, "%s")' % (
int(inode), int(id_package), parent_path, name, int(size),
int(ctime), int(crtime), int(atime), int(mtime),
extracted_file_path)
SQLite3.execute_commit_query(query, preprocess_db_path)
def parsing_updated_package(updated_package, preprocess_db_path):
has_updated = 1
package_name = updated_package.get('name')
code_path = updated_package.get('codePath')
if updated_package.get('dt'):
dt = int(updated_package.get('dt'), 16) / 1000
else:
dt = 0
ft = int(updated_package.get('ft'), 16) / 1000
it = int(updated_package.get('it'), 16) / 1000
ut = int(updated_package.get('ut'), 16) / 1000
version = int(updated_package.get('version'))
if updated_package.get('userId'):
uid_suid = int(updated_package.get('userId'))
is_suid = 0
elif updated_package.get('sharedUserId'):
uid_suid = int(updated_package.get('sharedUserId'))
is_suid = 1
ret_perms = SystemLog.get_package_permission(updated_package,
preprocess_db_path)
cnt_perms = ret_perms[0]
perms_value = str(ret_perms[1])
flag_del = 0
query = 'UPDATE package_info set has_updated = %d WHERE uid_suid = %d' % (
int(has_updated), int(uid_suid))
SQLite3.execute_commit_query(query, preprocess_db_path)
query = 'INSERT INTO updated_package_info(flag_del, package_name, code_path, dt, ft, it, ut, version, uid_suid, is_suid, cnt_perms, perms) VALUES(%d, "%s", "%s", %d, %d, %d, %d, %d, %d, %d, %d, "%s")' % (
int(flag_del), package_name, code_path, int(dt), int(ft), int(it),
int(ut), int(version), int(uid_suid), int(is_suid), int(cnt_perms),
perms_value)
SQLite3.execute_commit_query(query, preprocess_db_path)
def set_file_format_table_to_loaddb(list_dic_file_format_signature, load_db_path):
for i in range(len(list_dic_file_format_signature)):
format_name = list_dic_file_format_signature[i]['format']
format_flag = i+1
query = 'SELECT count(*) FROM "file_format" WHERE format = "%s"' % format_name
if(SQLite3.execute_fetch_query(query, load_db_path)[0]) == 0:
query = 'INSERT INTO file_format(format, flag) VALUES("%s", %d)' % (format_name, format_flag)
SQLite3.execute_commit_query(query, load_db_path)
def set_file_format_table(file_name, load_db_path):
query = 'SELECT flag FROM file_format ORDER BY flag DESC LIMIT 1'
flag_last = SQLite3.execute_fetch_query(query, load_db_path)[0]
query = 'SELECT count(*) FROM "%s" WHERE format = "%s"' % ("file_format", file_name)
flag = '%d' % SQLite3.execute_fetch_query(query, load_db_path)
if flag == '0':
query = 'INSERT INTO file_format(format, flag) VALUES("%s", %d)' % (file_name, flag_last+1)
SQLite3.execute_commit_query(query, load_db_path)
def parsing_permissions(permissions, preprocess_db_path):
for item in permissions.iter('item'):
permission_name = item.get('name')
package_name = item.get('package')
if item.get('protection'):
protection = int(item.get('protection'))
else:
protection = 0
query = 'INSERT INTO permission_info(permission_name, package_name, protection) VALUES("%s", "%s", %d)' % (
permission_name, package_name, int(protection))
SQLite3.execute_commit_query(query, preprocess_db_path)
def classify_with_file_name(file_name, load_db_path): query = 'SELECT flag FROM file_format WHERE format = "%s"' % file_name flag_signature = SQLite3.execute_fetch_query(query, load_db_path)[0] if file_name == "SQLITEDB_JOURNAL": query = 'UPDATE tsk_files set format = %d WHERE name LIKE ' % flag_signature query2 = '"%-journal" and dir_type !=3 and dir_flags != 2 and type = 0 and size != 0' # print(query + query2) SQLite3.execute_commit_query(query + query2, load_db_path) elif file_name == "APK": query = 'UPDATE tsk_files set format = %d WHERE name LIKE ' % flag_signature query2 = '"%.apk" and dir_type !=3 and dir_flags != 2 and type = 0 and size != 0' SQLite3.execute_commit_query(query + query2, load_db_path) else: return 0
def do_compare(list_file_inode, image_file_path, size_buf, list_dic_file_format_signature, load_db_path, result):
for i in range(0, len(list_file_inode)):
inode = '%d' % list_file_inode.pop()
header = TSK.get_file_buffer(image_file_path, int(inode), size_buf)
header_hex = b2a_hex(header)
# insert_file_signature_to_loaddb
format_name = Classifier.compare_signature(header_hex, list_dic_file_format_signature)
if format_name != False:
query = 'SELECT flag FROM file_format WHERE format = "%s"' % format_name
flag_signature = SQLite3.execute_fetch_query(query, load_db_path)
if flag_signature != None:
query = 'UPDATE tsk_files set format = %d WHERE meta_addr = %d and dir_flags != 2 and type = 0' % (int(flag_signature[0]), int(inode))
# print("query: ", query)
SQLite3.execute_commit_query(query, load_db_path)
result.put(len(list_file_inode))
def get_permission_id(name, preprocess_db_path):
query = 'SELECT rowid FROM permission_info WHERE permission_name = "%s"' % name
ret = SQLite3.execute_fetch_query(query, preprocess_db_path)
if ret:
return ret[0]
else:
return False
def set_loaddb(load_db_path): # create a id_package column in loaddb's tsk_files table query = "SELECT sql FROM sqlite_master WHERE name='tsk_files' AND sql LIKE '%id_package%'" if(SQLite3.execute_fetch_query(query, load_db_path)) == None: query = "ALTER TABLE tsk_files ADD id_package INTEGER DEFAULT 0 NOT NULL" SQLite3.execute_commit_query(query, load_db_path) # create a format column in loaddb's tsk_files table query = "SELECT sql FROM sqlite_master WHERE name='tsk_files' AND sql LIKE '%format%'" if(SQLite3.execute_fetch_query(query, load_db_path)) == None: query = "ALTER TABLE tsk_files ADD format INTEGER DEFAULT 0 NOT NULL" SQLite3.execute_commit_query(query, load_db_path) # create a file_format table in loaddb query = 'SELECT count(*) FROM sqlite_master WHERE name = "file_format"' if SQLite3.execute_fetch_query(query, load_db_path)[0] == 0: query = "CREATE TABLE file_format (format TEXT, flag INTEGER)" SQLite3.execute_commit_query(query, load_db_path)
def create_analysis_db(self):
if os.path.exists(self.analysis_db_path):
return self.analysis_db_path
list_query = list()
query_create_application_list_table = "CREATE TABLE application_list (is_deleted INTEGER, category TEXT, package_name TEXT, app_name TEXT, version TEXT, installed_time INTEGER, apk_changed_time INTEGER, updated_time INTEGER, deleted_time INTEGER, fs_ctime INTEGER, fs_crtime INTEGER, fs_atime INTEGER, fs_mtime INTEGER, is_updated INTEGER, source TEXT)"
query_create_id_password_hash_table = "CREATE TABLE id_password_hash (package_name TEXT, url TEXT, account TEXT, pwd TEXT, contents TEXT, timestamp TEXT, source TEXT)"
query_create_call_history_table = "CREATE TABLE call_history (package_name TEXT, timestamp TEXT, time_duration TEXT, phonenumber TEXT, account TEXT, digit_positive TEXT, file TEXT, contents TEXT, source TEXT)"
query_create_geodata_table = "CREATE TABLE geodata (package_name TEXT, timestamp TEXT, geodata TEXT, file TEXT, contents TEXT, source TEXT)"
query_create_web_brwoser_history_table = "CREATE TABLE web_browser_history (package_name TEXT, timestamp TEXT, url TEXT, account TEXT, digit_positive TEXT, file TEXT, contents TEXT, source TEXT)"
query_create_file_history_table = "CREATE TABLE file_history (package_name TEXT, timestamp TEXT, file TEXT, phonenumber TEXT, account TEXT, contents TEXT, source TEXT)"
query_create_embedded_filetable = "CREATE TABLE embedded_file (is_compressed INTEGER, parent_path TEXT, name TEXT, extension TEXT, mod_time TEXT, size INTEGER, compressed_size INTEGER, CRC INTEGER, create_system TEXT, source_path TEXT, source TEXT)"
list_query.append(query_create_application_list_table)
list_query.append(query_create_id_password_hash_table)
list_query.append(query_create_call_history_table)
list_query.append(query_create_geodata_table)
list_query.append(query_create_web_brwoser_history_table)
list_query.append(query_create_file_history_table)
list_query.append(query_create_embedded_filetable)
SQLite3.execute_commit_query(list_query, self.analysis_db_path)
def parsing_aid_information(preprocess_db_path):
if not os.path.exists(AID_CONF_PATH):
logger.error('Not exist the config (\"%s\").' % AID_CONF_PATH)
return False
try:
f = open(AID_CONF_PATH, 'r')
except Exception as e:
logger.error("Fail to open file [%s]" % AID_CONF_PATH)
return False
lines = f.readlines()
for line in lines:
if line.startswith(AID_DEFINE_START):
args = line.split(' ')
aid_name = args[1]
aid = args[2]
query = 'INSERT INTO aid_info(aid_name, aid) VALUES("%s", %d)' % (
aid_name, int(aid))
SQLite3.execute_commit_query(query, preprocess_db_path)
f.close()
def parsing_package(package, preprocess_db_path):
package_name = package.get('name')
code_path = package.get('codePath')
app_name = ""
if code_path.startswith('/system/'):
app_name = code_path.split('/')[-1]
flag_private = 0
if package.get('privateFlags'):
flag_private = int(package.get('privateFlags'))
if package.get('dt'):
dt = int(package.get('dt'), 16) / 1000
else:
dt = 0
ft = int(package.get('ft'), 16) / 1000
it = int(package.get('it'), 16) / 1000
ut = int(package.get('ut'), 16) / 1000
version = int(package.get('version'))
if package.get('userId'):
uid_suid = int(package.get('userId'))
is_suid = 0
elif package.get('sharedUserId'):
uid_suid = int(package.get('sharedUserId'))
is_suid = 1
ret_perms = SystemLog.get_package_permission(package,
preprocess_db_path)
cnt_perms = ret_perms[0]
perms_value = str(ret_perms[1])
has_updated = 0
flag_del = 0
query = 'INSERT INTO package_info(flag_del, package_name, app_name, code_path, flag_private, dt, ft, it, ut, version, uid_suid, is_suid, cnt_perms, perms, has_updated) VALUES(%d, "%s", "%s", "%s", %d, %d, %d, %d, %d, %d, %d, %d, %d, "%s", %d)' % (
flag_del, package_name, app_name, code_path, int(flag_private),
int(dt), int(ft), int(it), int(ut), int(version), int(uid_suid),
int(is_suid), int(cnt_perms), perms_value, int(has_updated))
SQLite3.execute_commit_query(query, preprocess_db_path)
def parsing_packages_list(packages_list, preprocess_db_path):
try:
f = open(packages_list, 'r')
except Exception as e:
logger.error("Fail to open file [%s]" % packages_list)
return False
lines = f.readlines()
for line in lines:
args = line.split(' ')
cnt_args = len(args)
if cnt_args >= 4:
package_name = args[0]
uid_suid = args[1]
unknown = args[2]
log_path = args[3]
category = ""
cnt_aid = 0
aids = ""
if cnt_args >= 5:
category = args[4]
if category.find(":") != False:
category = category.split(":")[0]
if cnt_args >= 6:
aids = args[5]
if aids != 'none\n':
cnt_aid = len(aids.split(','))
else:
aids = ""
query = 'UPDATE package_info set log_path = "%s", category = "%s", cnt_aid = %d, aids = "%s" WHERE uid_suid = %d and package_name = "%s"' % (
log_path, category, int(cnt_aid), aids, int(uid_suid),
package_name)
SQLite3.execute_commit_query(query, preprocess_db_path)
f.close()
def create_preprocess_db(self):
if os.path.exists(self.preprocess_db_path):
return self.preprocess_db_path
list_query = list()
query_create_image_file_info_table = "CREATE TABLE image_file_info (image_file_path TEXT, size_image INTEGER, size_alloc_area INTEGER, size_unalloc_area INTEGER, size_metadata_area INTEGER, size_apks INTEGER, size_applogs INTEGER, size_sdcard_area INTEGER)"
query_create_permission_info_table = "CREATE TABLE permission_info (permission_name TEXT, package_name TEXT, protection INTEGER)"
query_create_package_info_table = "CREATE TABLE package_info (flag_del INTEGER, package_name TEXT, app_name TEXT, uid_suid INTEGER, is_suid INTEGER, code_path TEXT, log_path TEXT, version INTEGER, flag_private INTEGER, dt INTEGER, ft INTEGER, it INTEGER, ut INTEGER, cnt_perms INTEGER, perms TEXT, has_updated INTEGER, category TEXT, cnt_aid INTEGER, aids TEXT)"
query_create_updated_package_info_table = "CREATE TABLE updated_package_info (flag_del INTEGER, package_name TEXT, uid_suid INTEGER, is_suid INTEGER, code_path TEXT, version INTEGER, dt INTEGER, ft INTEGER, it INTEGER, ut INTEGER, cnt_perms INTEGER, perms TEXT)"
query_create_aid_info_table = "CREATE TABLE aid_info (aid_name TEXT, aid INTEGER)"
query_create_sqlitedb_info_table = "CREATE TABLE sqlitedb_info (inode INTEGER, id_package INTEGER DEFAULT 0 NOT NULL, parent_path TEXT, name TEXT, size INTEGER, ctime INTEGER, crtime INTEGER, atime INTEGER, mtime INTEGER, extracted_path TEXT)"
query_create_apk_file_info_table = "CREATE TABLE apk_file_info (inode INTEGER, id_package INTEGER DEFAULT 0 NOT NULL, parent_path TEXT, name TEXT, size INTEGER, ctime INTEGER, crtime INTEGER, atime INTEGER, mtime INTEGER, app_name TEXT, cnt_perms INTEGER, perms TEXT, sha1 TEXT, extracted_path TEXT)"
query_create_sqlitedb_table_preprocess_table = "CREATE TABLE sqlitedb_table_preprocess (inode INTEGER, table_name TEXT, cnt_records INTEGER, cnt_timestamp INTEGER, cnt_time_duration INTEGER, cnt_phonenumber INTEGER, cnt_account INTEGER, cnt_pwd INTEGER, cnt_url INTEGER, cnt_geodata INTEGER, cnt_ip INTEGER, cnt_mac INTEGER, cnt_digit_positive INTEGER, cnt_contents INTEGER, cnt_bin INTEGER, cnt_file INTEGER, cnt_cipher INTEGER, cnt_pkg INTEGER, timestamp TEXT, time_duration TEXT, phonenumber TEXT, account TEXT, pwd TEXT, url TEXT, geodata TEXT, ip TEXT, mac TEXT, digit_positive TEXT, contents TEXT, bin TEXT, file TEXT, cipher TEXT, pkg TEXT, last_col INTEGER)"
list_query.append(query_create_image_file_info_table)
list_query.append(query_create_permission_info_table)
list_query.append(query_create_package_info_table)
list_query.append(query_create_updated_package_info_table)
list_query.append(query_create_aid_info_table)
list_query.append(query_create_sqlitedb_info_table)
list_query.append(query_create_apk_file_info_table)
list_query.append(query_create_sqlitedb_table_preprocess_table)
SQLite3.execute_commit_query(list_query, self.preprocess_db_path)
def get_userinfo_type_format_value_from_sqlitedb(dic_userinfo_record,
dic_userinfo_cnt,
not_null_userinfo_type):
list_userinfo_type = list()
list_col_name = list()
list_value_format = list()
list_idx_col = list()
list_not_null_col = list()
for userinfo_type, cnt_userinfo in dic_userinfo_cnt.items():
for i in range(cnt_userinfo):
list_userinfo_type.append(userinfo_type)
userinfo_record = dic_userinfo_record[userinfo_type].split(
",")[i]
list_col_name.append(userinfo_record.split("(")[0])
col_name = userinfo_record.split("(")[0]
if userinfo_type == not_null_userinfo_type:
list_not_null_col.append(col_name)
list_value_format.append(
userinfo_record.split("(")[1].replace(")", ""))
list_idx_col.append(
userinfo_record.split("(")[2].replace(")", ""))
query_select_contents = ""
if len(list_col_name) == 1:
query_select_contents += list_col_name[0]
else:
for col_name in list_col_name:
query_select_contents += "\"" + col_name + "\", "
query_select_contents = query_select_contents[:-2]
query_where_not_null = ""
for not_null_col in list_not_null_col:
query_where_not_null += not_null_col + " !=\"\" or "
query_where_not_null = query_where_not_null[:-3]
query = 'SELECT DISTINCT %s FROM \"%s\" WHERE %s' % (
query_select_contents, dic_userinfo_record['table_name'],
query_where_not_null)
list_userinfo_col_value = SQLite3.execute_fetch_query_multi_values(
query, dic_userinfo_record['db_path'])
return list_userinfo_type, list_value_format, list_col_name, list_userinfo_col_value