def _parse(self, f, options={}):
print "Parsing APK"
information = []
zipfile = ZipFile(f)
# print zipfile.namelist()
# exit(0)
skip_android_assets = False
if 'AndroidManifest.xml' in zipfile.namelist():
resources_file = None
manifest_file = zipfile.read('AndroidManifest.xml')
if 'resources.arsc' in zipfile.namelist():
resources_file = zipfile.read('resources.arsc')
information += self.parse_manifest(manifest_file, resources_file)
skip_android_assets = True
# print information
for x in zipfile.infolist():
# Add File to index
if skip_android_assets and (x.filename == 'AndroidManifest.xml'
or x.filename == 'resources.arsc'):
continue
information.append(
self.createData("main",
"FILE",
FILE_NAME=x.filename,
FILE_CHECKSUM=x.CRC))
# Get only dex and so files for now -- TODO remove this after caching
if x.filename.endswith('.dex'):
# or x.filename.endswith('.so'):
print "Parsing %s" % x.filename
# Process File
parser = P.instance().get_parser(x.filename)
if parser is None:
continue
try:
results = parser.parse(zipfile.open(x.filename),
parent=self.filename_w)
if results is not None:
information += results
except Exception as e:
print "Cannot process %s" % x.filename
import traceback
print(traceback.format_exc())
return information
def _parse(self, f, options={}):
print "Parsing ZIP"
return []
information = []
zipfile = ZipFile(f)
# try:
for x in zipfile.infolist():
# Add File to index
data = self.createData("strings", "FILE", filename=x.filename)
data["CRC"] = x.CRC
information.append(data)
# Process File
parser = P.instance().get_parser(x.filename)
if parser is None:
continue
try:
before = datetime.datetime.now()
results = parser.parse(zipfile.open(x.filename),
parent=self.filename_w)
after = datetime.datetime.now()
if (after - before) > datetime.timedelta(seconds=1):
print "Parsed %s in %s" % (x.filename, str(after - before))
if results is not None:
information += results
except Exception as e:
print "Cannot process %s" % x.filename
print e
return information
def test_dex(self):
results = Parser().get_parser(dir_path + "/samples/classes.dex")
print len(results.parse()), "Modules Found"
def test_version(self):
results = Parser().get_parser(
dir_path + "/samples/android.arch.lifecycle_viewmodel.version")
print len(results.parse()), "Modules Found"
def test_prop(self):
results = Parser().get_parser(dir_path +
"/samples/build-data.properties")
print len(results.parse()), "Modules Found"
def test_apk(self):
results = Parser().get_parser(dir_path + "/samples/pango.apk")
print len(results.parse()), "Modules Found"
def test_so(self):
results = Parser().get_parser(dir_path +
"/samples/libJniBitmapOperator.so")
print len(results.parse()), "Modules Found"
def test_arsc(self):
results = Parser().get_parser(dir_path + "/samples/resources.arsc")
print len(results.parse()), "Modules Found"
def test_manifest_xml(self):
results = Parser().get_parser(dir_path +
"/samples/AndroidManifest.xml")
print len(results.parse()), "Modules Found"
# encoding: utf-8
import os
from libsearch.processing.searchparser import Parser
from libsearch.storage.indexer import Indexer
import traceback
import datetime
dir_path = os.path.dirname(os.path.realpath(__file__)) + "/../cache"
files = os.listdir(dir_path)
for f in files:
if f.endswith(".apk"):
if os.path.isfile(dir_path + os.sep + f + '.indexed'):
print "Already indexed: %s" % (dir_path + os.sep + f)
continue
try:
parser = Parser.instance().get_parser(dir_path + os.sep + f)
if parser != None:
index_data = parser.parse(save=True)
# Indexer.instance().save(index_data)
# with open(dir_path + os.sep + f + '.indexed', 'w') as cached:
# cached.write(str(datetime.date.today()))
except Exception:
print(traceback.format_exc())
pass
print "Done"
import os
from libsearch.processing.searchparser import Parser
from libsearch.storage.indexer import Indexer
from libsearch.fetcher import apk_downloader
import urllib3
urllib3.disable_warnings()
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--csv', help='csv file with apps')
parser.add_argument('--download',
help='download apk first',
action="store_true")
args = parser.parse_args()
if args.csv is not None:
with open(args.csv) as f:
apps = ["/en/" + line.strip() for line in f]
for apk in apps:
try:
link_download = apk_downloader(apk, download=args.download)
if link_download is not None:
apk_parser = Parser.instance().parsers['apk'](apk)
apk_parser.parse(fileobj=link_download, save=True)
except Exception as e:
print "failed downloading %s" % apk
print e
pass