def ios_source_analysis(src): """iOS Objective-C Code Analysis""" try: logger.info("Starting iOS Source Code and PLIST Analysis") api_rules = ios_apis.CODE_APIS code_rules = ios_rules.CODE_RULES code_findings = {} api_findings = {} email_n_file = [] url_n_file = [] url_list = [] domains = {} for dirname, _, files in os.walk(src): for jfile in files: if jfile.endswith(".m"): jfile_path = os.path.join(src, dirname, jfile) if "+" in jfile: new_path = os.path.join(src, dirname, jfile.replace("+", "x")) shutil.move(jfile_path, new_path) jfile_path = new_path dat = '' with io.open(jfile_path, mode='r', encoding="utf8", errors="ignore") as flip: dat = flip.read() # Code Analysis relative_src_path = jfile_path.replace(src, '') code_rule_matcher(code_findings, [], dat, relative_src_path, code_rules) # API Analysis api_rule_matcher(api_findings, [], dat, relative_src_path, api_rules) # Extract URLs and Emails urls, urls_nf, emails_nf = url_n_email_extract( dat, relative_src_path) url_list.extend(urls) url_n_file.extend(urls_nf) email_n_file.extend(emails_nf) # Domain Extraction and Malware Check logger.info("Performing Malware Check on extracted Domains") domains = malware_check(list(set(url_list))) logger.info("Finished Code Analysis, Email and URL Extraction") code_analysis_dict = { 'api': api_findings, 'code_anal': code_findings, 'urlnfile': url_n_file, 'domains': domains, 'emailnfile': email_n_file, } return code_analysis_dict except: PrintException("iOS Source Code Analysis")
def ios_source_analysis(src): """iOS Objective-C Code Analysis""" try: logger.info("Starting iOS Source Code and PLIST Analysis") api_rules = ios_apis.CODE_APIS code_rules = ios_rules.CODE_RULES code_findings = {} api_findings = {} email_n_file = [] url_n_file = [] url_list = [] domains = {} for dirname, _, files in os.walk(src): for jfile in files: if jfile.endswith(".m"): jfile_path = os.path.join(src, dirname, jfile) if "+" in jfile: new_path = os.path.join( src, dirname, jfile.replace("+", "x")) shutil.move(jfile_path, new_path) jfile_path = new_path dat = '' with io.open(jfile_path, mode='r', encoding="utf8", errors="ignore") as flip: dat = flip.read() # Code Analysis relative_src_path = jfile_path.replace(src, '') code_rule_matcher(code_findings, [], dat, relative_src_path, code_rules) # API Analysis api_rule_matcher(api_findings, [], dat, relative_src_path, api_rules) # Extract URLs and Emails urls, urls_nf, emails_nf = url_n_email_extract(dat, relative_src_path) url_list.extend(urls) url_n_file.extend(urls_nf) email_n_file.extend(emails_nf) urls_list = list(set(url_list)) # Domain Extraction and Malware Check logger.info("Performing Malware Check on extracted Domains") domains = malware_check(urls_list) logger.info("Finished Code Analysis, Email and URL Extraction") code_analysis_dict = { 'api': api_findings, 'code_anal': code_findings, 'urls_list': urls_list, 'urlnfile': url_n_file, 'domains': domains, 'emailnfile': email_n_file, } return code_analysis_dict except: PrintException("iOS Source Code Analysis")
def run_analysis(apk_dir, md5_hash, package): """Run Dynamic File Analysis.""" analysis_result = {} logger.info('Dynamic File Analysis') domains = {} clipboard = [] # Collect Log data datas = get_log_data(apk_dir, package) clip_tag = 'I/CLIPDUMP-INFO-LOG' clip_tag2 = 'I CLIPDUMP-INFO-LOG' # Collect Clipboard for log_line in datas['logcat']: if clip_tag in log_line: clipboard.append(log_line.replace(clip_tag, 'Process ID ')) if clip_tag2 in log_line: log_line = log_line.split(clip_tag2)[1] clipboard.append(log_line) # URLs My Custom regex url_pattern = re.compile( r'((?:https?://|s?ftps?://|file://|' r'javascript:|data:|www\d{0,3}' r'[.])[\w().=/;,#:@?&~*+!$%\'{}-]+)', re.UNICODE) urls = re.findall(url_pattern, datas['traffic'].lower()) if urls: urls = list(set(urls)) else: urls = [] # Domain Extraction and Malware Check logger.info('Performing Malware Check on extracted Domains') domains = malware_check(urls) # Email Etraction Regex emails = [] regex = re.compile(r'[\w.-]+@[\w-]+\.[\w]{2,}') for email in regex.findall(datas['traffic'].lower()): if (email not in emails) and (not email.startswith('//')): emails.append(email) # Tar dump and fetch files all_files = get_app_files(apk_dir, md5_hash, package) analysis_result['urls'] = urls analysis_result['domains'] = domains analysis_result['emails'] = emails analysis_result['clipboard'] = clipboard analysis_result['xml'] = all_files['xml'] analysis_result['sqlite'] = all_files['sqlite'] analysis_result['other_files'] = all_files['others'] return analysis_result
def static_analyzer(request, api=False): """Do static analysis on an request and save to db.""" try: if api: typ = request.POST['scan_type'] checksum = request.POST['hash'] filename = request.POST['file_name'] rescan = str(request.POST.get('re_scan', 0)) else: typ = request.GET['type'] checksum = request.GET['checksum'] filename = request.GET['name'] rescan = str(request.GET.get('rescan', 0)) # Input validation app_dic = {} match = re.match('^[0-9a-f]{32}$', checksum) if ((match) and (filename.lower().endswith('.apk') or filename.lower().endswith('.zip')) and (typ in ['zip', 'apk'])): app_dic['dir'] = settings.BASE_DIR # BASE DIR app_dic['app_name'] = filename # APP ORGINAL NAME app_dic['md5'] = checksum # MD5 app_dic['app_dir'] = os.path.join( settings.UPLD_DIR, app_dic['md5'] + '/') # APP DIRECTORY app_dic['tools_dir'] = os.path.join( app_dic['dir'], 'StaticAnalyzer/tools/') # TOOLS DIR # DWD_DIR = settings.DWD_DIR # not needed? Var is never used. logger.info("Starting Analysis on : " + app_dic['app_name']) if typ == 'apk': # Check if in DB # pylint: disable=E1101 db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) else: app_dic[ 'app_file'] = app_dic['md5'] + '.apk' # NEW FILENAME app_dic['app_path'] = app_dic['app_dir'] + \ app_dic['app_file'] # APP PATH # ANALYSIS BEGINS app_dic['size'] = str(file_size( app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic['sha256'] = hash_gen( app_dic['app_path']) app_dic['files'] = unzip(app_dic['app_path'], app_dic['app_dir']) if not app_dic['files']: # Can't Analyze APK, bail out. msg = "APK file is invalid or corrupt" if api: return print_n_send_error_response( request, msg, True) else: return print_n_send_error_response( request, msg, False) app_dic['certz'] = get_hardcoded_cert_keystore( app_dic['files']) logger.info("APK Extracted") # Manifest XML app_dic['parsed_xml'] = get_manifest( app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir'], '', True) # Get icon res_path = os.path.join(app_dic['app_dir'], 'res') app_dic['icon_hidden'] = True # Even if the icon is hidden, try to guess it by the # default paths app_dic['icon_found'] = False app_dic['icon_path'] = '' # TODO: Check for possible different names for resource # folder? if os.path.exists(res_path): icon_dic = get_icon(app_dic['app_path'], res_path) if icon_dic: app_dic['icon_hidden'] = icon_dic['hidden'] app_dic['icon_found'] = bool(icon_dic['path']) app_dic['icon_path'] = icon_dic['path'] # Set Manifest link app_dic['mani'] = '../ManifestView/?md5=' + \ app_dic['md5'] + '&type=apk&bin=1' man_data_dic = manifest_data(app_dic['parsed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis(app_dic['parsed_xml'], man_data_dic) bin_an_buff = [] bin_an_buff += elf_analysis(app_dic['app_dir']) bin_an_buff += res_analysis(app_dic['app_dir']) cert_dic = cert_info(app_dic['app_dir'], app_dic['app_file'], app_dic['tools_dir']) apkid_results = apkid_analysis(app_dic['app_dir'], app_dic['app_path'], app_dic['app_name']) tracker = Trackers.Trackers(app_dic['app_dir'], app_dic['tools_dir']) tracker_res = tracker.get_trackers() dex_2_jar(app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir']) dex_2_smali(app_dic['app_dir'], app_dic['tools_dir']) jar_2_java(app_dic['app_dir'], app_dic['tools_dir']) code_an_dic = code_analysis(app_dic['app_dir'], man_an_dic['permissons'], "apk") # Get the strings string_res = strings_jar(app_dic['app_file'], app_dic['app_dir']) if string_res: app_dic['strings'] = string_res['strings'] code_an_dic["urls_list"].extend( string_res['urls_list']) code_an_dic["urls"].extend(string_res['url_nf']) code_an_dic["emails"].extend(string_res['emails_nf']) else: app_dic['strings'] = [] # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic["urls_list"]))) # Domain Extraction and Malware Check logger.info( "Performing Malware Check on extracted Domains") code_an_dic["domains"] = malware_check( list(set(code_an_dic["urls_list"]))) logger.info("Generating Java and Smali Downloads") gen_downloads(app_dic['app_dir'], app_dic['md5'], app_dic['icon_path']) app_dic['zipped'] = '&type=apk' logger.info("Connecting to Database") try: # SAVE TO DB if rescan == '1': logger.info("Updating Database...") update_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info("Saving to Database") create_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) except: PrintException("Saving to Database Failed") context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) context["average_cvss"], context["security_score"] = score( context["findings"]) context['dynamic_analysis_done'] = os.path.exists( os.path.join(app_dic['app_dir'], 'logcat.txt')) context['VT_RESULT'] = None if settings.VT_ENABLED: vt = VirusTotal.VirusTotal() context['VT_RESULT'] = vt.get_result( os.path.join(app_dic['app_dir'], app_dic['md5']) + '.apk', app_dic['md5']) template = "static_analysis/android_binary_analysis.html" if api: return context else: return render(request, template, context) elif typ == 'zip': # Check if in DB # pylint: disable=E1101 cert_dic = {} cert_dic['cert_info'] = '' cert_dic['issued'] = '' cert_dic['sha256Digest'] = False bin_an_buff = [] app_dic['strings'] = '' app_dic['zipped'] = '' # Above fields are only available for APK and not ZIP db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) else: app_dic[ 'app_file'] = app_dic['md5'] + '.zip' # NEW FILENAME app_dic['app_path'] = app_dic['app_dir'] + \ app_dic['app_file'] # APP PATH logger.info("Extracting ZIP") app_dic['files'] = unzip(app_dic['app_path'], app_dic['app_dir']) # Check if Valid Directory Structure and get ZIP Type pro_type, valid = valid_android_zip(app_dic['app_dir']) if valid and pro_type == 'ios': logger.info("Redirecting to iOS Source Code Analyzer") if api: return {"type": "ios"} else: return HttpResponseRedirect( '/StaticAnalyzer_iOS/?name=' + app_dic['app_name'] + '&type=ios&checksum=' + app_dic['md5']) app_dic['certz'] = get_hardcoded_cert_keystore( app_dic['files']) app_dic['zipped'] = pro_type logger.info("ZIP Type - " + pro_type) if valid and (pro_type in ['eclipse', 'studio']): # ANALYSIS BEGINS app_dic['size'] = str(file_size( app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic['sha256'] = hash_gen( app_dic['app_path']) # Manifest XML app_dic['persed_xml'] = get_manifest( "", app_dic['app_dir'], app_dic['tools_dir'], pro_type, False) # Set manifest view link app_dic['mani'] = ('../ManifestView/?md5=' + app_dic['md5'] + '&type=' + pro_type + '&bin=0') man_data_dic = manifest_data(app_dic['persed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis(app_dic['persed_xml'], man_data_dic) # Get icon eclipse_res_path = os.path.join( app_dic['app_dir'], 'res') studio_res_path = os.path.join(app_dic['app_dir'], 'app', 'src', 'main', 'res') if os.path.exists(eclipse_res_path): res_path = eclipse_res_path elif os.path.exists(studio_res_path): res_path = studio_res_path else: res_path = '' app_dic['icon_hidden'] = man_an_dic['icon_hidden'] app_dic['icon_found'] = False app_dic['icon_path'] = '' if res_path: app_dic['icon_path'] = find_icon_path_zip( res_path, man_data_dic['icons']) if app_dic['icon_path']: app_dic['icon_found'] = True if app_dic['icon_path']: if os.path.exists(app_dic['icon_path']): shutil.copy2( app_dic['icon_path'], os.path.join(settings.DWD_DIR, app_dic['md5'] + '-icon.png')) code_an_dic = code_analysis(app_dic['app_dir'], man_an_dic['permissons'], pro_type) # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic["urls_list"]))) # Domain Extraction and Malware Check logger.info( "Performing Malware Check on extracted Domains") code_an_dic["domains"] = malware_check( list(set(code_an_dic["urls_list"]))) logger.info("Connecting to Database") try: # SAVE TO DB if rescan == '1': logger.info("Updating Database...") update_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info("Saving to Database") create_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) except: PrintException("Saving to Database Failed") context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) else: msg = "This ZIP Format is not supported" if api: return print_n_send_error_response( request, msg, True) else: print_n_send_error_response(request, msg, False) return HttpResponseRedirect('/zip_format/') context["average_cvss"], context["security_score"] = score( context["findings"]) template = "static_analysis/android_source_analysis.html" if api: return context else: return render(request, template, context) else: logger.error( "Only APK,IPA and Zipped Android/iOS Source code supported now!" ) else: msg = "Hash match failed or Invalid file extension or file type" if api: return print_n_send_error_response(request, msg, True) else: return print_n_send_error_response(request, msg, False) except Exception as excep: msg = str(excep) exp = excep.__doc__ if api: return print_n_send_error_response(request, msg, True, exp) else: return print_n_send_error_response(request, msg, False, exp)
def code_analysis(app_dir, perms, typ): """Perform the code analysis.""" try: logger.info("Static Android Code Analysis Started") api_rules = android_apis.APIS code_rules = android_rules.RULES code_findings = {} api_findings = {} email_n_file = [] url_n_file = [] url_list = [] domains = {} if typ == "apk": java_src = os.path.join(app_dir, 'java_source/') elif typ == "studio": java_src = os.path.join(app_dir, 'app/src/main/java/') elif typ == "eclipse": java_src = os.path.join(app_dir, 'src/') logger.info("Code Analysis Started on - " + java_src) # pylint: disable=unused-variable # Needed by os.walk for dir_name, sub_dir, files in os.walk(java_src): for jfile in files: jfile_path = os.path.join(java_src, dir_name, jfile) if "+" in jfile: p_2 = os.path.join(java_src, dir_name, jfile.replace("+", "x")) shutil.move(jfile_path, p_2) jfile_path = p_2 repath = dir_name.replace(java_src, '') if (jfile.endswith('.java') and any( re.search(cls, repath) for cls in settings.SKIP_CLASSES) is False): dat = '' with io.open(jfile_path, mode='r', encoding="utf8", errors="ignore") as file_pointer: dat = file_pointer.read() # Code Analysis # print "[INFO] Doing Code Analysis on - " + jfile_path relative_java_path = jfile_path.replace(java_src, '') code_rule_matcher(code_findings, list(perms.keys()), dat, relative_java_path, code_rules) # API Check api_rule_matcher(api_findings, list(perms.keys()), dat, relative_java_path, api_rules) # Extract URLs and Emails urls, urls_nf, emails_nf = url_n_email_extract( dat, relative_java_path) url_list.extend(urls) url_n_file.extend(urls_nf) email_n_file.extend(emails_nf) # Domain Extraction and Malware Check logger.info("Performing Malware Check on extracted Domains") domains = malware_check(list(set(url_list))) logger.info("Finished Code Analysis, Email and URL Extraction") code_an_dic = { 'api': api_findings, 'findings': code_findings, 'urls': url_n_file, 'domains': domains, 'emails': email_n_file, } return code_an_dic except: PrintException("Performing Code Analysis")
def static_analyzer(request, api=False): """Do static analysis on an request and save to db.""" try: if api: typ = request.POST['scan_type'] checksum = request.POST['hash'] filename = request.POST['file_name'] rescan = str(request.POST.get('re_scan', 0)) else: typ = request.GET['type'] checksum = request.GET['checksum'] filename = request.GET['name'] rescan = str(request.GET.get('rescan', 0)) # Input validation app_dic = {} match = re.match('^[0-9a-f]{32}$', checksum) if ((match) and (filename.lower().endswith('.apk') or filename.lower().endswith('.zip')) and (typ in ['zip', 'apk'])): app_dic['dir'] = Path(settings.BASE_DIR) # BASE DIR app_dic['app_name'] = filename # APP ORGINAL NAME app_dic['md5'] = checksum # MD5 # APP DIRECTORY app_dic['app_dir'] = Path(settings.UPLD_DIR) / checksum app_dic['tools_dir'] = app_dic['dir'] / 'StaticAnalyzer' / 'tools' app_dic['tools_dir'] = app_dic['tools_dir'].as_posix() logger.info('Starting Analysis on : %s', app_dic['app_name']) if typ == 'apk': app_dic['app_file'] = app_dic['md5'] + '.apk' # NEW FILENAME app_dic['app_path'] = (app_dic['app_dir'] / app_dic['app_file']).as_posix() app_dic['app_dir'] = app_dic['app_dir'].as_posix() + '/' # Check if in DB # pylint: disable=E1101 db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) else: # ANALYSIS BEGINS app_dic['size'] = str(file_size( app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic['sha256'] = hash_gen( app_dic['app_path']) app_dic['files'] = unzip(app_dic['app_path'], app_dic['app_dir']) if not app_dic['files']: # Can't Analyze APK, bail out. msg = 'APK file is invalid or corrupt' if api: return print_n_send_error_response( request, msg, True) else: return print_n_send_error_response( request, msg, False) app_dic['certz'] = get_hardcoded_cert_keystore( app_dic['files']) logger.info('APK Extracted') # Manifest XML app_dic['parsed_xml'] = get_manifest( app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir'], '', True, ) # get app_name app_dic['real_name'] = get_app_name( app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir'], True, ) # Get icon res_path = os.path.join(app_dic['app_dir'], 'res') app_dic['icon_hidden'] = True # Even if the icon is hidden, try to guess it by the # default paths app_dic['icon_found'] = False app_dic['icon_path'] = '' # TODO: Check for possible different names for resource # folder? if os.path.exists(res_path): icon_dic = get_icon(app_dic['app_path'], res_path) if icon_dic: app_dic['icon_hidden'] = icon_dic['hidden'] app_dic['icon_found'] = bool(icon_dic['path']) app_dic['icon_path'] = icon_dic['path'] # Set Manifest link app_dic['mani'] = ('../ManifestView/?md5=' + app_dic['md5'] + '&type=apk&bin=1') man_data_dic = manifest_data(app_dic['parsed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis( app_dic['parsed_xml'], man_data_dic, '', app_dic['app_dir'], ) bin_an_buff = [] bin_an_buff += elf_analysis(app_dic['app_dir']) bin_an_buff += res_analysis(app_dic['app_dir']) cert_dic = cert_info(app_dic['app_dir'], app_dic['app_file']) apkid_results = apkid_analysis(app_dic['app_dir'], app_dic['app_path'], app_dic['app_name']) tracker = Trackers.Trackers(app_dic['app_dir'], app_dic['tools_dir']) tracker_res = tracker.get_trackers() apk_2_java(app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir']) dex_2_smali(app_dic['app_dir'], app_dic['tools_dir']) code_an_dic = code_analysis(app_dic['app_dir'], 'apk') # Get the strings string_res = strings_jar(app_dic['app_file'], app_dic['app_dir']) if string_res: app_dic['strings'] = string_res['strings'] app_dic['secrets'] = string_res['secrets'] code_an_dic['urls_list'].extend( string_res['urls_list']) code_an_dic['urls'].extend(string_res['url_nf']) code_an_dic['emails'].extend(string_res['emails_nf']) else: app_dic['strings'] = [] app_dic['secrets'] = [] # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic['urls_list']))) # Domain Extraction and Malware Check logger.info( 'Performing Malware Check on extracted Domains') code_an_dic['domains'] = malware_check( list(set(code_an_dic['urls_list']))) # Copy App icon copy_icon(app_dic['md5'], app_dic['icon_path']) app_dic['zipped'] = 'apk' logger.info('Connecting to Database') try: # SAVE TO DB if rescan == '1': logger.info('Updating Database...') save_or_update( 'update', app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info('Saving to Database') save_or_update( 'save', app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) except Exception: logger.exception('Saving to Database Failed') context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, tracker_res, ) context['average_cvss'], context['security_score'] = score( context['code_analysis']) context['dynamic_analysis_done'] = is_file_exists( os.path.join(app_dic['app_dir'], 'logcat.txt')) context['virus_total'] = None if settings.VT_ENABLED: vt = VirusTotal.VirusTotal() context['virus_total'] = vt.get_result( app_dic['app_path'], app_dic['md5']) template = 'static_analysis/android_binary_analysis.html' if api: return context else: return render(request, template, context) elif typ == 'zip': ios_ret = HttpResponseRedirect('/StaticAnalyzer_iOS/?name=' + app_dic['app_name'] + '&type=ios&checksum=' + app_dic['md5']) # Check if in DB # pylint: disable=E1101 cert_dic = { 'certificate_info': '', 'certificate_status': '', 'description': '', } bin_an_buff = [] app_dic['strings'] = [] app_dic['secrets'] = [] app_dic['zipped'] = '' # Above fields are only available for APK and not ZIP app_dic['app_file'] = app_dic['md5'] + '.zip' # NEW FILENAME app_dic['app_path'] = (app_dic['app_dir'] / app_dic['app_file']).as_posix() app_dic['app_dir'] = app_dic['app_dir'].as_posix() + '/' db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) ios_db_entry = StaticAnalyzerIOS.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) elif ios_db_entry.exists() and rescan == '0': if api: return {'type': 'ios'} else: return ios_ret else: logger.info('Extracting ZIP') app_dic['files'] = unzip(app_dic['app_path'], app_dic['app_dir']) # Check if Valid Directory Structure and get ZIP Type pro_type, valid = valid_android_zip(app_dic['app_dir']) if valid and pro_type == 'ios': logger.info('Redirecting to iOS Source Code Analyzer') if api: return {'type': 'ios'} else: return ios_ret app_dic['certz'] = get_hardcoded_cert_keystore( app_dic['files']) app_dic['zipped'] = pro_type logger.info('ZIP Type - %s', pro_type) if valid and (pro_type in ['eclipse', 'studio']): # ANALYSIS BEGINS app_dic['size'] = str(file_size( app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic['sha256'] = hash_gen( app_dic['app_path']) # Manifest XML app_dic['persed_xml'] = get_manifest( '', app_dic['app_dir'], app_dic['tools_dir'], pro_type, False, ) # get app_name app_dic['real_name'] = get_app_name( app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir'], False, ) # Set manifest view link app_dic['mani'] = ('../ManifestView/?md5=' + app_dic['md5'] + '&type=' + pro_type + '&bin=0') man_data_dic = manifest_data(app_dic['persed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis( app_dic['persed_xml'], man_data_dic, pro_type, app_dic['app_dir'], ) # Get icon eclipse_res_path = os.path.join( app_dic['app_dir'], 'res') studio_res_path = os.path.join(app_dic['app_dir'], 'app', 'src', 'main', 'res') if os.path.exists(eclipse_res_path): res_path = eclipse_res_path elif os.path.exists(studio_res_path): res_path = studio_res_path else: res_path = '' app_dic['icon_hidden'] = man_an_dic['icon_hidden'] app_dic['icon_found'] = False app_dic['icon_path'] = '' if res_path: app_dic['icon_path'] = find_icon_path_zip( res_path, man_data_dic['icons']) if app_dic['icon_path']: app_dic['icon_found'] = True if app_dic['icon_path']: if os.path.exists(app_dic['icon_path']): shutil.copy2( app_dic['icon_path'], os.path.join(settings.DWD_DIR, app_dic['md5'] + '-icon.png')) code_an_dic = code_analysis(app_dic['app_dir'], pro_type) # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic['urls_list']))) # Domain Extraction and Malware Check logger.info( 'Performing Malware Check on extracted Domains') code_an_dic['domains'] = malware_check( list(set(code_an_dic['urls_list']))) logger.info('Connecting to Database') try: # SAVE TO DB if rescan == '1': logger.info('Updating Database...') save_or_update( 'update', app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info('Saving to Database') save_or_update( 'save', app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) except Exception: logger.exception('Saving to Database Failed') context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, {}, ) else: msg = 'This ZIP Format is not supported' if api: return print_n_send_error_response( request, msg, True) else: print_n_send_error_response(request, msg, False) ctx = { 'title': 'Invalid ZIP archive', 'version': settings.MOBSF_VER, } template = 'general/zip.html' return render(request, template, ctx) context['average_cvss'], context['security_score'] = score( context['code_analysis']) template = 'static_analysis/android_source_analysis.html' if api: return context else: return render(request, template, context) else: err = ('Only APK,IPA and Zipped ' 'Android/iOS Source code supported now!') logger.error(err) else: msg = 'Hash match failed or Invalid file extension or file type' if api: return print_n_send_error_response(request, msg, True) else: return print_n_send_error_response(request, msg, False) except Exception as excep: logger.exception('Error Performing Static Analysis') msg = str(excep) exp = excep.__doc__ if api: return print_n_send_error_response(request, msg, True, exp) else: return print_n_send_error_response(request, msg, False, exp)
def ios_source_analysis(src): """IOS Objective-C and Swift Code Analysis.""" try: logger.info('Starting iOS Source Code and PLIST Analysis') root = Path(settings.BASE_DIR) / 'StaticAnalyzer' / 'views' swift_rules = root / 'ios' / 'rules' / 'swift_rules.yaml' objective_c_rules = root / 'ios' / 'rules' / 'objective_c_rules.yaml' api_rules = root / 'ios' / 'rules' / 'ios_apis.yaml' code_findings = {} api_findings = {} email_n_file = [] url_n_file = [] url_list = [] domains = {} source_type = '' source_types = set() # Code and API Analysis code_findings = scan(objective_c_rules.as_posix(), {'.m'}, [src], settings.SKIP_CLASS_PATH) if code_findings: source_types.add(_SourceType.objc) code_findings.update( scan(swift_rules.as_posix(), {'.swift'}, [src], settings.SKIP_CLASS_PATH)) if code_findings: source_types.add(_SourceType.swift) api_findings = scan(api_rules.as_posix(), {'.m', '.swift'}, [src], settings.SKIP_CLASS_PATH) # Extract URLs and Emails skp = settings.SKIP_CLASS_PATH for pfile in Path(src).rglob('*'): if ((pfile.suffix in ('.m', '.swift') and any(skip_path in pfile.as_posix() for skip_path in skp) is False)): relative_java_path = pfile.as_posix().replace(src, '') urls, urls_nf, emails_nf = url_n_email_extract( pfile.read_text('utf-8', 'ignore'), relative_java_path) url_list.extend(urls) url_n_file.extend(urls_nf) email_n_file.extend(emails_nf) if not source_types: source_type = _SourceType.nocode.value elif len(source_types) > 1: source_type = _SourceType.swift_and_objc.value else: source_type = source_types.pop().value urls_list = list(set(url_list)) # Domain Extraction and Malware Check logger.info('Performing Malware Check on extracted Domains') domains = malware_check(urls_list) logger.info('Finished Code Analysis, Email and URL Extraction') code_analysis_dict = { 'api': api_findings, 'code_anal': code_findings, 'urls_list': urls_list, 'urlnfile': url_n_file, 'domains': domains, 'emailnfile': email_n_file, 'source_type': source_type, } return code_analysis_dict except Exception: logger.exception('iOS Source Code Analysis')
def static_analyzer(request, api=False): """Do static analysis on an request and save to db.""" try: if api: typ = request.POST['scan_type'] checksum = request.POST['hash'] filename = request.POST['file_name'] rescan = str(request.POST.get('re_scan', 0)) else: typ = request.GET['type'] checksum = request.GET['checksum'] filename = request.GET['name'] rescan = str(request.GET.get('rescan', 0)) # Input validation app_dic = {} match = re.match('^[0-9a-f]{32}$', checksum) if ( ( match ) and ( filename.lower().endswith('.apk') or filename.lower().endswith('.zip') ) and ( typ in ['zip', 'apk'] ) ): app_dic['dir'] = settings.BASE_DIR # BASE DIR app_dic['app_name'] = filename # APP ORGINAL NAME app_dic['md5'] = checksum # MD5 app_dic['app_dir'] = os.path.join(settings.UPLD_DIR, app_dic[ 'md5'] + '/') # APP DIRECTORY app_dic['tools_dir'] = os.path.join( app_dic['dir'], 'StaticAnalyzer/tools/') # TOOLS DIR # DWD_DIR = settings.DWD_DIR # not needed? Var is never used. logger.info("Starting Analysis on : " + app_dic['app_name']) if typ == 'apk': # Check if in DB # pylint: disable=E1101 db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) else: app_dic['app_file'] = app_dic[ 'md5'] + '.apk' # NEW FILENAME app_dic['app_path'] = app_dic['app_dir'] + \ app_dic['app_file'] # APP PATH # ANALYSIS BEGINS app_dic['size'] = str( file_size(app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic[ 'sha256'] = hash_gen(app_dic['app_path']) app_dic['files'] = unzip( app_dic['app_path'], app_dic['app_dir']) app_dic['certz'] = get_hardcoded_cert_keystore(app_dic[ 'files']) logger.info("APK Extracted") # Manifest XML app_dic['parsed_xml'] = get_manifest( app_dic['app_path'], app_dic['app_dir'], app_dic['tools_dir'], '', True ) # Get icon res_path = os.path.join(app_dic['app_dir'], 'res') app_dic['icon_hidden'] = True # Even if the icon is hidden, try to guess it by the # default paths app_dic['icon_found'] = False app_dic['icon_path'] = '' # TODO: Check for possible different names for resource # folder? if os.path.exists(res_path): icon_dic = get_icon( app_dic['app_path'], res_path) if icon_dic: app_dic['icon_hidden'] = icon_dic['hidden'] app_dic['icon_found'] = bool(icon_dic['path']) app_dic['icon_path'] = icon_dic['path'] # Set Manifest link app_dic['mani'] = '../ManifestView/?md5=' + \ app_dic['md5'] + '&type=apk&bin=1' man_data_dic = manifest_data(app_dic['parsed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis( app_dic['parsed_xml'], man_data_dic ) bin_an_buff = [] bin_an_buff += elf_analysis(app_dic['app_dir']) bin_an_buff += res_analysis(app_dic['app_dir']) cert_dic = cert_info( app_dic['app_dir'], app_dic['app_file'], app_dic['tools_dir']) apkid_results = apkid_analysis(app_dic[ 'app_dir'], app_dic['app_path'], app_dic['app_name']) dex_2_jar(app_dic['app_path'], app_dic[ 'app_dir'], app_dic['tools_dir']) dex_2_smali(app_dic['app_dir'], app_dic['tools_dir']) jar_2_java(app_dic['app_dir'], app_dic['tools_dir']) code_an_dic = code_analysis( app_dic['app_dir'], man_an_dic['permissons'], "apk" ) # Get the strings string_res = strings_jar( app_dic['app_file'], app_dic['app_dir'] ) if string_res: app_dic['strings'] = string_res['strings'] code_an_dic["urls_list"].extend( string_res['urls_list']) code_an_dic["urls"].extend(string_res['url_nf']) code_an_dic["emails"].extend(string_res['emails_nf']) else: app_dic['strings'] = [] # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic["urls_list"]))) # Domain Extraction and Malware Check logger.info( "Performing Malware Check on extracted Domains") code_an_dic["domains"] = malware_check( list(set(code_an_dic["urls_list"]))) logger.info("Generating Java and Smali Downloads") gen_downloads(app_dic['app_dir'], app_dic[ 'md5'], app_dic['icon_path']) app_dic['zipped'] = '&type=apk' logger.info("Connecting to Database") try: # SAVE TO DB if rescan == '1': logger.info("Updating Database...") update_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info("Saving to Database") create_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, ) except: PrintException("Saving to Database Failed") context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, apkid_results, ) context["average_cvss"], context[ "security_score"] = score(context["findings"]) context['dynamic_analysis_done'] = os.path.exists( os.path.join(app_dic['app_dir'], 'logcat.txt')) context['VT_RESULT'] = None if settings.VT_ENABLED: vt = VirusTotal.VirusTotal() context['VT_RESULT'] = vt.get_result( os.path.join(app_dic['app_dir'], app_dic['md5']) + '.apk', app_dic['md5'] ) template = "static_analysis/android_binary_analysis.html" if api: return context else: return render(request, template, context) elif typ == 'zip': # Check if in DB # pylint: disable=E1101 cert_dic = {} cert_dic['cert_info'] = '' cert_dic['issued'] = '' cert_dic['sha256Digest'] = False bin_an_buff = [] app_dic['strings'] = '' app_dic['zipped'] = '' # Above fields are only available for APK and not ZIP db_entry = StaticAnalyzerAndroid.objects.filter( MD5=app_dic['md5']) if db_entry.exists() and rescan == '0': context = get_context_from_db_entry(db_entry) else: app_dic['app_file'] = app_dic[ 'md5'] + '.zip' # NEW FILENAME app_dic['app_path'] = app_dic['app_dir'] + \ app_dic['app_file'] # APP PATH logger.info("Extracting ZIP") app_dic['files'] = unzip( app_dic['app_path'], app_dic['app_dir']) # Check if Valid Directory Structure and get ZIP Type pro_type, valid = valid_android_zip(app_dic['app_dir']) if valid and pro_type == 'ios': logger.info("Redirecting to iOS Source Code Analyzer") if api: return {"type": "ios"} else: return HttpResponseRedirect( '/StaticAnalyzer_iOS/?name=' + app_dic['app_name'] + '&type=ios&checksum=' + app_dic['md5'] ) app_dic['certz'] = get_hardcoded_cert_keystore(app_dic[ 'files']) app_dic['zipped'] = pro_type logger.info("ZIP Type - " + pro_type) if valid and (pro_type in ['eclipse', 'studio']): # ANALYSIS BEGINS app_dic['size'] = str( file_size(app_dic['app_path'])) + 'MB' # FILE SIZE app_dic['sha1'], app_dic[ 'sha256'] = hash_gen(app_dic['app_path']) # Manifest XML app_dic['persed_xml'] = get_manifest( "", app_dic['app_dir'], app_dic['tools_dir'], pro_type, False ) # Set manifest view link app_dic['mani'] = ( '../ManifestView/?md5=' + app_dic['md5'] + '&type=' + pro_type + '&bin=0' ) man_data_dic = manifest_data(app_dic['persed_xml']) app_dic['playstore'] = get_app_details( man_data_dic['packagename']) man_an_dic = manifest_analysis( app_dic['persed_xml'], man_data_dic ) # Get icon eclipse_res_path = os.path.join( app_dic['app_dir'], 'res') studio_res_path = os.path.join( app_dic['app_dir'], 'app', 'src', 'main', 'res') if os.path.exists(eclipse_res_path): res_path = eclipse_res_path elif os.path.exists(studio_res_path): res_path = studio_res_path else: res_path = '' app_dic['icon_hidden'] = man_an_dic['icon_hidden'] app_dic['icon_found'] = False app_dic['icon_path'] = '' if res_path: app_dic['icon_path'] = find_icon_path_zip( res_path, man_data_dic['icons']) if app_dic['icon_path']: app_dic['icon_found'] = True if app_dic['icon_path']: if os.path.exists(app_dic['icon_path']): shutil.copy2(app_dic['icon_path'], os.path.join( settings.DWD_DIR, app_dic['md5'] + '-icon.png')) code_an_dic = code_analysis( app_dic['app_dir'], man_an_dic['permissons'], pro_type ) # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( list(set(code_an_dic["urls_list"]))) # Domain Extraction and Malware Check logger.info( "Performing Malware Check on extracted Domains") code_an_dic["domains"] = malware_check( list(set(code_an_dic["urls_list"]))) logger.info("Connecting to Database") try: # SAVE TO DB if rescan == '1': logger.info("Updating Database...") update_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, ) update_scan_timestamp(app_dic['md5']) elif rescan == '0': logger.info("Saving to Database") create_db_entry( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, ) except: PrintException("Saving to Database Failed") context = get_context_from_analysis( app_dic, man_data_dic, man_an_dic, code_an_dic, cert_dic, bin_an_buff, {}, ) else: msg = "This ZIP Format is not supported" if api: return print_n_send_error_response(request, msg, True) else: print_n_send_error_response(request, msg, False) return HttpResponseRedirect('/zip_format/') context["average_cvss"], context[ "security_score"] = score(context["findings"]) template = "static_analysis/android_source_analysis.html" if api: return context else: return render(request, template, context) else: logger.error( "Only APK,IPA and Zipped Android/iOS Source code supported now!") else: msg = "Hash match failed or Invalid file extension or file type" if api: return print_n_send_error_response(request, msg, True) else: return print_n_send_error_response(request, msg, False) except Exception as excep: msg = str(excep) exp = excep.__doc__ if api: return print_n_send_error_response(request, msg, True, exp) else: return print_n_send_error_response(request, msg, False, exp)
def ios_source_analysis(src): """IOS Objective-C and Swift Code Analysis.""" try: logger.info('Starting iOS Source Code and PLIST Analysis') code_findings = {} api_findings = {} email_n_file = [] url_n_file = [] url_list = [] domains = {} source_type = '' source_types = set() for dirname, _, files in os.walk(src): for jfile in files: if jfile.endswith('.m'): api_rules = ios_apis.CODE_APIS code_rules = objc_rules.OBJC_RULES source_types.add(_SourceType.objc) elif jfile.endswith('.swift'): api_rules = ios_apis.CODE_APIS code_rules = swift_rules.SWIFT_RULES source_types.add(_SourceType.swift) else: continue jfile_path = os.path.join(src, dirname, jfile) if '+' in jfile: new_path = os.path.join( src, dirname, jfile.replace('+', 'x')) shutil.move(jfile_path, new_path) jfile_path = new_path dat = '' with io.open(jfile_path, mode='r', encoding='utf8', errors='ignore') as flip: dat = flip.read() # Code Analysis relative_src_path = jfile_path.replace(src, '') code_rule_matcher(code_findings, [], dat, relative_src_path, code_rules) # API Analysis api_rule_matcher(api_findings, [], dat, relative_src_path, api_rules) # Extract URLs and Emails urls, urls_nf, emails_nf = url_n_email_extract( dat, relative_src_path) url_list.extend(urls) url_n_file.extend(urls_nf) email_n_file.extend(emails_nf) if not source_types: source_type = _SourceType.nocode.value elif len(source_types) > 1: source_type = _SourceType.swift_and_objc.value else: source_type = source_types.pop().value urls_list = list(set(url_list)) # Domain Extraction and Malware Check logger.info('Performing Malware Check on extracted Domains') domains = malware_check(urls_list) logger.info('Finished Code Analysis, Email and URL Extraction') code_analysis_dict = { 'api': api_findings, 'code_anal': code_findings, 'urls_list': urls_list, 'urlnfile': url_n_file, 'domains': domains, 'emailnfile': email_n_file, 'source_type': source_type, } return code_analysis_dict except Exception: logger.exception('iOS Source Code Analysis')
def run_analysis(apk_dir, md5_hash, package): """Run Dynamic File Analysis.""" analysis_result = {} logger.info('Dynamic File Analysis') capfuzz_home = os.path.join(str(Path.home()), '.capfuzz') web = os.path.join(capfuzz_home, 'flows', package + '.flows.txt') logcat = os.path.join(apk_dir, 'logcat.txt') xlogcat = os.path.join(apk_dir, 'x_logcat.txt') traffic = '' web_data = '' xlg = '' domains = {} logcat_data = [] clipboard = [] clip_tag = 'I/CLIPDUMP-INFO-LOG' try: with io.open(web, mode='r', encoding='utf8', errors='ignore') as flip: web_data = flip.read() except Exception: pass with io.open(logcat, mode='r', encoding='utf8', errors='ignore') as flip: logcat_data = flip.readlines() traffic = ''.join(logcat_data) with io.open(xlogcat, mode='r', encoding='utf8', errors='ignore') as flip: xlg = flip.read() traffic = web_data + traffic + xlg for log_line in logcat_data: if log_line.startswith(clip_tag): clipboard.append(log_line.replace(clip_tag, 'Process ID ')) urls = [] # URLs My Custom regex url_pattern = re.compile( r'((?:https?://|s?ftps?://|file://|' r'javascript:|data:|www\d{0,3}' r'[.])[\w().=/;,#:@?&~*+!$%\'{}-]+)', re.UNICODE) urllist = re.findall(url_pattern, traffic.lower()) # Domain Extraction and Malware Check logger.info('Performing Malware Check on extracted Domains') domains = malware_check(urllist) for url in urllist: if url not in urls: urls.append(url) # Email Etraction Regex emails = [] regex = re.compile(r'[\w.-]+@[\w-]+\.[\w.]+') for email in regex.findall(traffic.lower()): if (email not in emails) and (not email.startswith('//')): if email == '*****@*****.**': pass else: emails.append(email) # Extract Device Data try: tar_loc = os.path.join(apk_dir, package + '.tar') untar_dir = os.path.join(apk_dir, 'DYNAMIC_DeviceData/') if not os.path.exists(untar_dir): os.makedirs(untar_dir) with tarfile.open(tar_loc) as tar: try: tar.extractall(untar_dir) except Exception: pass except Exception: logger.exception('TAR EXTRACTION FAILED') # Do Static Analysis on Data from Device xmlfiles = '' sqlite_db = '' other_files = '' typ = '' untar_dir = os.path.join(apk_dir, 'DYNAMIC_DeviceData/') if not os.path.exists(untar_dir): os.makedirs(untar_dir) try: for dir_name, _, files in os.walk(untar_dir): for jfile in files: file_path = os.path.join(untar_dir, dir_name, jfile) if '+' in file_path: shutil.move(file_path, file_path.replace('+', 'x')) file_path = file_path.replace('+', 'x') fileparam = file_path.replace(untar_dir, '') if jfile == 'lib': pass else: if jfile.endswith('.xml'): typ = 'xml' xmlfiles += ('<tr><td><a href="../View/?' 'file={}&md5={}&type={}">{}' '</a></td><tr>').format( escape(fileparam), md5_hash, typ, escape(fileparam)) else: with open(file_path, 'r') as flip: file_cnt_sig = flip.read(6) if file_cnt_sig == 'SQLite': typ = 'db' sqlite_db += ('<tr><td><a href="../View/?' 'file={}&md5={}&type={}">{}' '</a></td><tr>').format( escape(fileparam), md5_hash, typ, escape(fileparam)) elif not jfile.endswith('.DS_Store'): typ = 'others' other_files += ('<tr><td><a href="../View/?' 'file={}&md5={}&type={}">{}' '</a></td><tr>').format( escape(fileparam), md5_hash, typ, escape(fileparam)) except Exception: logger.exception('Dynamic File Analysis') analysis_result['urls'] = urls analysis_result['domains'] = domains analysis_result['emails'] = emails analysis_result['clipboard'] = clipboard analysis_result['web_data'] = web_data analysis_result['xmlfiles'] = xmlfiles analysis_result['sqlite_db'] = sqlite_db analysis_result['other_files'] = other_files return analysis_result
def run_analysis(apk_dir, md5_hash, package): """Run Dynamic File Analysis""" analysis_result = {} logger.info("Dynamic File Analysis") capfuzz_home = os.path.join(str(Path.home()), ".capfuzz") web = os.path.join(capfuzz_home, 'flows', package + ".flows.txt") logcat = os.path.join(apk_dir, 'logcat.txt') xlogcat = os.path.join(apk_dir, 'x_logcat.txt') traffic = '' web_data = '' xlg = '' domains = {} logcat_data = [] clipboard = [] clip_tag = "I/CLIPDUMP-INFO-LOG" try: with io.open(web, mode='r', encoding="utf8", errors="ignore") as flip: web_data = flip.read() except: pass with io.open(logcat, mode='r', encoding="utf8", errors="ignore") as flip: logcat_data = flip.readlines() traffic = ''.join(logcat_data) with io.open(xlogcat, mode='r', encoding="utf8", errors="ignore") as flip: xlg = flip.read() traffic = web_data + traffic + xlg for log_line in logcat_data: if log_line.startswith(clip_tag): clipboard.append(log_line.replace(clip_tag, "Process ID ")) urls = [] # URLs My Custom regex url_pattern = re.compile( r'((?:https?://|s?ftps?://|file://|javascript:|data:|www\d{0,3}[.])[\w().=/;,#:@?&~*+!$%\'{}-]+)', re.UNICODE) urllist = re.findall(url_pattern, traffic.lower()) # Domain Extraction and Malware Check logger.info("Performing Malware Check on extracted Domains") domains = malware_check(urllist) for url in urllist: if url not in urls: urls.append(url) # Email Etraction Regex emails = [] regex = re.compile(r"[\w.-]+@[\w-]+\.[\w.]+") for email in regex.findall(traffic.lower()): if (email not in emails) and (not email.startswith('//')): if email == "*****@*****.**": pass else: emails.append(email) # Extract Device Data try: tar_loc = os.path.join(apk_dir, package + '.tar') untar_dir = os.path.join(apk_dir, 'DYNAMIC_DeviceData/') if not os.path.exists(untar_dir): os.makedirs(untar_dir) with tarfile.open(tar_loc) as tar: try: tar.extractall(untar_dir) except: pass except: PrintException("TAR EXTRACTION FAILED") # Do Static Analysis on Data from Device xmlfiles = '' sqlite_db = '' other_files = '' typ = '' untar_dir = os.path.join(apk_dir, 'DYNAMIC_DeviceData/') if not os.path.exists(untar_dir): os.makedirs(untar_dir) try: for dir_name, _, files in os.walk(untar_dir): for jfile in files: file_path = os.path.join(untar_dir, dir_name, jfile) if "+" in file_path: shutil.move(file_path, file_path.replace("+", "x")) file_path = file_path.replace("+", "x") fileparam = file_path.replace(untar_dir, '') if jfile == 'lib': pass else: if jfile.endswith('.xml'): typ = 'xml' xmlfiles += "<tr><td><a href='../View/?file=" + \ escape(fileparam) + "&md5=" + md5_hash + "&type=" + \ typ + "'>" + escape(fileparam) + "</a></td><tr>" else: with io.open(file_path, mode='r', encoding="utf8", errors="ignore") as flip: file_cnt_sig = flip.read(6) if file_cnt_sig == "SQLite": typ = 'db' sqlite_db += "<tr><td><a href='../View/?file=" + \ escape(fileparam) + "&md5=" + md5_hash + "&type=" + \ typ + "'>" + \ escape(fileparam) + "</a></td><tr>" elif not jfile.endswith('.DS_Store'): typ = 'others' other_files += "<tr><td><a href='../View/?file=" + \ escape(fileparam) + "&md5=" + md5_hash + "&type=" + \ typ + "'>" + \ escape(fileparam) + "</a></td><tr>" except: PrintException("Dynamic File Analysis") analysis_result["urls"] = urls analysis_result["domains"] = domains analysis_result["emails"] = emails analysis_result["clipboard"] = clipboard analysis_result["web_data"] = web_data analysis_result["xmlfiles"] = xmlfiles analysis_result["sqlite_db"] = sqlite_db analysis_result["other_files"] = other_files return analysis_result