def _parse(self, f, options={}): print "Parsing APK" information = [] zipfile = ZipFile(f) # print zipfile.namelist() # exit(0) skip_android_assets = False if 'AndroidManifest.xml' in zipfile.namelist(): resources_file = None manifest_file = zipfile.read('AndroidManifest.xml') if 'resources.arsc' in zipfile.namelist(): resources_file = zipfile.read('resources.arsc') information += self.parse_manifest(manifest_file, resources_file) skip_android_assets = True # print information for x in zipfile.infolist(): # Add File to index if skip_android_assets and (x.filename == 'AndroidManifest.xml' or x.filename == 'resources.arsc'): continue information.append( self.createData("main", "FILE", FILE_NAME=x.filename, FILE_CHECKSUM=x.CRC)) # Get only dex and so files for now -- TODO remove this after caching if x.filename.endswith('.dex'): # or x.filename.endswith('.so'): print "Parsing %s" % x.filename # Process File parser = P.instance().get_parser(x.filename) if parser is None: continue try: results = parser.parse(zipfile.open(x.filename), parent=self.filename_w) if results is not None: information += results except Exception as e: print "Cannot process %s" % x.filename import traceback print(traceback.format_exc()) return information
def _parse(self, f, options={}): print "Parsing ZIP" return [] information = [] zipfile = ZipFile(f) # try: for x in zipfile.infolist(): # Add File to index data = self.createData("strings", "FILE", filename=x.filename) data["CRC"] = x.CRC information.append(data) # Process File parser = P.instance().get_parser(x.filename) if parser is None: continue try: before = datetime.datetime.now() results = parser.parse(zipfile.open(x.filename), parent=self.filename_w) after = datetime.datetime.now() if (after - before) > datetime.timedelta(seconds=1): print "Parsed %s in %s" % (x.filename, str(after - before)) if results is not None: information += results except Exception as e: print "Cannot process %s" % x.filename print e return information
def test_dex(self): results = Parser().get_parser(dir_path + "/samples/classes.dex") print len(results.parse()), "Modules Found"
def test_version(self): results = Parser().get_parser( dir_path + "/samples/android.arch.lifecycle_viewmodel.version") print len(results.parse()), "Modules Found"
def test_prop(self): results = Parser().get_parser(dir_path + "/samples/build-data.properties") print len(results.parse()), "Modules Found"
def test_apk(self): results = Parser().get_parser(dir_path + "/samples/pango.apk") print len(results.parse()), "Modules Found"
def test_so(self): results = Parser().get_parser(dir_path + "/samples/libJniBitmapOperator.so") print len(results.parse()), "Modules Found"
def test_arsc(self): results = Parser().get_parser(dir_path + "/samples/resources.arsc") print len(results.parse()), "Modules Found"
def test_manifest_xml(self): results = Parser().get_parser(dir_path + "/samples/AndroidManifest.xml") print len(results.parse()), "Modules Found"
# encoding: utf-8 import os from libsearch.processing.searchparser import Parser from libsearch.storage.indexer import Indexer import traceback import datetime dir_path = os.path.dirname(os.path.realpath(__file__)) + "/../cache" files = os.listdir(dir_path) for f in files: if f.endswith(".apk"): if os.path.isfile(dir_path + os.sep + f + '.indexed'): print "Already indexed: %s" % (dir_path + os.sep + f) continue try: parser = Parser.instance().get_parser(dir_path + os.sep + f) if parser != None: index_data = parser.parse(save=True) # Indexer.instance().save(index_data) # with open(dir_path + os.sep + f + '.indexed', 'w') as cached: # cached.write(str(datetime.date.today())) except Exception: print(traceback.format_exc()) pass print "Done"
import os from libsearch.processing.searchparser import Parser from libsearch.storage.indexer import Indexer from libsearch.fetcher import apk_downloader import urllib3 urllib3.disable_warnings() import argparse parser = argparse.ArgumentParser() parser.add_argument('--csv', help='csv file with apps') parser.add_argument('--download', help='download apk first', action="store_true") args = parser.parse_args() if args.csv is not None: with open(args.csv) as f: apps = ["/en/" + line.strip() for line in f] for apk in apps: try: link_download = apk_downloader(apk, download=args.download) if link_download is not None: apk_parser = Parser.instance().parsers['apk'](apk) apk_parser.parse(fileobj=link_download, save=True) except Exception as e: print "failed downloading %s" % apk print e pass