def __init__(self):
self.datahandlers = {
DataTypes.DATA: self.datahandler,
DataTypes.DATE: self.datehandler,
DataTypes.TEXT: self.texthandler
}
self.handlers = {
ExtractStore.TYPE_TABLE: self._get_tablehtml,
ExtractStore.TYPE_DATA: self._get_datahtml,
ExtractStore.TYPE_XML: self._get_xmlhtml,
ExtractStore.TYPE_IMAGE: self._get_imagehtml,
ExtractStore.TYPE_STRING: self._get_asciistringhtml
}
self.mime = MimeGuesser()
def __init__(self,
outqueue=None,
extract_store=None,
mountpoint=None,
settings=None,
versions=None):
self.mimguess = MimeGuesser()
self.outqueue = outqueue
self.settings = settings
self.versions = versions
imp = Importer()
app_modules = imp.get_package_modules("AndroidApps", IApp())
for app in app_modules:
name = app.get_packagename()
if name in self.appstore:
self.appstore[name].append(app)
else:
self.appstore[name] = [app]
if extract_store != None:
self.exstore = extract_store
else:
self.exstore = ExtractStore.ExtractStore()
def __init__(self):
self.datahandlers = {
DataTypes.DATA: self.datahandler,
DataTypes.DATE: self.datehandler,
DataTypes.TEXT: self.texthandler
}
self.handlers = {
ExtractStore.TYPE_TABLE: self._get_tablehtml,
ExtractStore.TYPE_DATA: self._get_datahtml,
ExtractStore.TYPE_XML: self._get_xmlhtml,
ExtractStore.TYPE_IMAGE: self._get_imagehtml,
ExtractStore.TYPE_STRING: self._get_asciistringhtml
}
self.mime = MimeGuesser()
def __init__(self, outqueue=None, extract_store=None, mountpoint=None,
settings=None, versions=None ):
self.mimguess = MimeGuesser()
self.outqueue = outqueue
self.settings = settings
self.versions = versions
imp = Importer()
app_modules = imp.get_package_modules("AndroidApps", IApp())
for app in app_modules:
name = app.get_packagename()
if name in self.appstore:
self.appstore[name].append(app)
else:
self.appstore[name] = [app]
if extract_store != None:
self.exstore = extract_store
else:
self.exstore = ExtractStore.ExtractStore()
class HtmlExtract(object):
def __init__(self):
self.datahandlers = {
DataTypes.DATA: self.datahandler,
DataTypes.DATE: self.datehandler,
DataTypes.TEXT: self.texthandler
}
self.handlers = {
ExtractStore.TYPE_TABLE: self._get_tablehtml,
ExtractStore.TYPE_DATA: self._get_datahtml,
ExtractStore.TYPE_XML: self._get_xmlhtml,
ExtractStore.TYPE_IMAGE: self._get_imagehtml,
ExtractStore.TYPE_STRING: self._get_asciistringhtml
}
self.mime = MimeGuesser()
def dump_file(self, appfile):
html = ""
if appfile.ftype == ExtractStore.TYPE_MULTI:
navhead = """
<div class="panel-heading">
<ul class="nav nav-tabs">
"""
navbody = """<div class="panel-body"><div class="tab-content">"""
for contents in appfile.content:
navhead += """<li><a data-toggle="tab" href="#{name}">{name}</a></li>""".format(name=contents.name)
navbody += """<div id={name} class="tab-pane">""".format(name=contents.name)
handler = None
try:
handler=self.handlers[contents.ctype]
except:
navbody += "No Handler Implemented"
#print "Bad Handler: " + appfile.name
if handler != None:
navbody+=handler(contents.content,contents.tbl_info,contents.knowninfo)
navbody += "</div>"
navhead += "</ul></div>" #ul and panel-heading
navbody += "</div></div>"
html += """<div class="panel with-nav-tabs panel-default">
""" + navhead + navbody + """</div>"""
#do the nav-tabs init here and add items
else:
handler = None
try:
handler = self.handlers[appfile.ftype]
except:
#print "No Handler: " + appfile.name
html += "Error No Handler Implemented"
if handler != None:
html += handler(appfile.content, appfile.tbl_info)
#html = html.encode("UTF-8")
return html
def dump_hex(self,appfile):
if appfile.ftype == ExtractStore.TYPE_MULTI:
if appfile.filepath == None:
return "<p>Unexpected Error: no filepath specified</p>"
try:
f = open(appfile.filepath, "rb")
content = f.read()
except:
return """
<p>File {} not found. Please make sure that you haven't
unmounted any images regarding this case
""".format(cgi.escape(appfile.filepath))
return self._get_datahtml(content)
else:
return self._get_datahtml(appfile.content)
def dump_strings(self, appfile):
if appfile.ftype == ExtractStore.TYPE_MULTI:
if appfile.filepath == None:
return "<p>Unexpected Error: no filepath specified</p>"
try:
f = open(appfile.filepath, "rb")
content = f.read()
except:
return """
<p>File {} not found. Please make sure that you haven't
unmounted any images regarding this case
""".format(cgi.escape(appfile.filepath))
return self._get_stringshtml( self.restringsdump(content))
else:
return self._get_stringshtml(self.restringsdump(appfile.content))
def datahandler(self, value):
return """
<button class="btn btn-primary btn-sm" onclick="hideshow(this);">
Show Data <span class="glyphicon glyphicon-chevron-down"></span>
</button>
<pre style="display:none;">{hexdump}</pre>
""".format(hexdump=self.hexdump(value))
def datehandler(self, value):
return """<span class="date">{value}</span>""".format(value=value)
def texthandler(self,value):
print "TEXTHANDLER CALLED"
return "Unimplemented"
def hexdump(self, src, length=16):
if src == None:
return "None"
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)])
lines = []
for c in xrange(0, len(src), length):
chars = src[c:c+length]
mhex = ' '.join(["%02x" % ord(x) for x in chars])
printable = cgi.escape(''.join(["%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.') for x in chars]))
lines.append("%04x %-*s %s\n" % (c, length*3, mhex, printable))
return ''.join(lines)
def restringsdump(self, content):
for match in re.finditer("[^\x00-\x1F\x7F-\xFF]{4,}", content):
yield( match.start(), cgi.escape(match.group()) )
def stringsdump(self, content, minlen=4):
result = ""
loc = 0
for c in content:
loc += 1
if c in string.printable:
result += c
continue
elif len(result) >= minlen:
yield (loc, cgi.escape(result))
result = ""
def _parse_column(self, colname, value, tblinfo=None,knowntable=None,notd=False):
ret = u""
if isinstance( value, StringIO.StringIO ):
value = value.buf
if notd:
valwrap = u"{value}"
else:
valwrap = u"<td>{value}</td>"
knownfields = None
if knowntable != None:
knownfields = knowntable.knownfields
if tblinfo != None:
if colname in tblinfo:
if tblinfo[colname] == "BLOB":
if value == None:
return valwrap.format(value="None")
mimetype = self.mime.get_buffermimetype(str(value), True)
if mimetype != None:
if mimetype.startswith("image/"):
return valwrap.format(value=self._get_imagehtml(value, {"mime":mimetype}))
elif mimetype.startswith("bplist"):
return valwrap.format(value=self._get_bplisthtml(value))
return valwrap.format(value=self.datahandler(value))
try:
if knownfields != None:
if colname in knownfields:
return valwrap.format(value=self.datahandlers[knownfields[colname]](value))
return valwrap.format(value=value)
except Exception, e:
if type(value) == StringIO:
value = value.buf
print traceback.format_exc()
print "colname: {colname}".format(colname=colname)
sys.stderr.write(repr(e) + "\n")
# <button type="button" class="btn btn-primary btn-sm">Small button</button>
return valwrap.format(value=self.datahandler(value))
return ret
class ApplicationParser(object):
parser_name = "Android Application Parser"
appstore = {}
exstore = None
def __init__(self,
outqueue=None,
extract_store=None,
mountpoint=None,
settings=None,
versions=None):
self.mimguess = MimeGuesser()
self.outqueue = outqueue
self.settings = settings
self.versions = versions
imp = Importer()
app_modules = imp.get_package_modules("AndroidApps", IApp())
for app in app_modules:
name = app.get_packagename()
if name in self.appstore:
self.appstore[name].append(app)
else:
self.appstore[name] = [app]
if extract_store != None:
self.exstore = extract_store
else:
self.exstore = ExtractStore.ExtractStore()
def get_extractedstore(self):
return self.exstore
def get_package_list(self):
return self.appstore.keys()
def get_package(self, name):
if name not in self.appstore:
return None
if len(self.appstore[name]) == 1:
if self.appstore[name][0].has_defaultversion() == False:
self.outqueue.put("Package {} is not default".format(name))
return self.appstore[name][0]
"""
If multiple entries for the same package exist, then we need to decide
which one we should use
"""
default = None
appversion = self.versions.get_application_version(name)
if appversion == None or appversion == "N/A":
self.outqueue.put(
"{} could not determine App version".format(name))
else:
try:
appversion = int(appversion)
except:
pass
for defn in self.appstore[name]:
if defn.has_defaultversion:
if default != None:
self.outqueue.put(
"{} has more than one default".format(name))
default = defn
if defn.has_version(appversion):
return defn
return default
def handle_file(self, filename, dirpath, app):
filepath = join(dirpath, filename)
handler = self.mimguess.get_handler(filepath)
knowninfo = app.get_file_info(filename)
fobject = None
if handler != None:
fobject = handler(filename, filepath, knowninfo, self.outqueue,
self.settings)
else:
"""
This is a remote case where no handler can handle the file.
However, the default handler for unknown files should always be the
raw data handler
"""
fobject = ExtractStore.ApplicationFile(filename,
ExtractStore.TYPE_NONE)
fobject.set_mime(self.mimguess.get_filemime(filepath))
fobject.add_sig(genfilesig(filepath, DEFAULT_HASHER))
fobject.add_stats(os.stat(filepath))
return fobject
def scan_single_app_dir(self, dirroot, app, dirobject, store_app):
for f in listdir(dirroot):
filepath = join(dirroot, f)
if isdir(filepath):
subdirobject = dirobject.add_dir(f)
self.scan_single_app_dir(filepath, app, subdirobject,
store_app)
elif islink(filepath):
"""Skip for now"""
continue
elif isfile(filepath):
try:
dirobject.add_file(self.handle_file(f, dirroot, app))
store_app.totalfiles += 1
except:
print("Error handling file %(fname)s" %
{'fname': join(dirroot, f)})
return
def read_link(self, linkpath):
target_path = os.readlink(linkpath)
if os.path.isabs(target_path):
return target_path
return join(dirname(linkpath), target_path)
def scan_single_app(self, approot_path, app):
#print "[APP]: " + app.get_packagename()
if self.outqueue != None:
self.outqueue.put("[APP]: " + app.get_packagename())
store_app = self.exstore.create_application(app.get_packagename(),
app.get_canonicalname())
rootdir = store_app.add_directory(approot_path)
store_app.add_root_stats(os.stat(approot_path))
for f in listdir(approot_path):
filepath = join(approot_path, f)
if isdir(filepath):
dirobject = rootdir.add_dir(f)
self.scan_single_app_dir(filepath, app, dirobject, store_app)
elif islink(filepath):
if filepath.endswith("lib"):
store_app.set_library_path(self.read_link(filepath))
elif isfile(filepath):
store_app.totalfiles += 1
rootdir.add_file(self.handle_file(f, approot_path, app))
return
def scan_directory(self, root):
for dfile in os.listdir(root):
fqname = join(root, dfile)
if isdir(fqname):
app = self.get_package(dfile)
if app == None:
app = IApp()
app.set_packagename(dfile)
self.scan_single_app(fqname, app)
else:
print "Stray file found: " + fqname
#self.handle_file(fqname, None)
if self.outqueue != None:
self.outqueue.put("Scanned Total Apps: " +
str(len(self.exstore.store)))
class HtmlExtract(object):
def __init__(self):
self.datahandlers = {
DataTypes.DATA: self.datahandler,
DataTypes.DATE: self.datehandler,
DataTypes.TEXT: self.texthandler
}
self.handlers = {
ExtractStore.TYPE_TABLE: self._get_tablehtml,
ExtractStore.TYPE_DATA: self._get_datahtml,
ExtractStore.TYPE_XML: self._get_xmlhtml,
ExtractStore.TYPE_IMAGE: self._get_imagehtml,
ExtractStore.TYPE_STRING: self._get_asciistringhtml
}
self.mime = MimeGuesser()
def dump_file(self, appfile):
html = ""
if appfile.ftype == ExtractStore.TYPE_MULTI:
navhead = """
<div class="panel-heading">
<ul class="nav nav-tabs">
"""
navbody = """<div class="panel-body"><div class="tab-content">"""
for contents in appfile.content:
navhead += """<li><a data-toggle="tab" href="#{name}">{name}</a></li>""".format(
name=contents.name)
navbody += """<div id={name} class="tab-pane">""".format(
name=contents.name)
handler = None
try:
handler = self.handlers[contents.ctype]
except:
navbody += "No Handler Implemented"
#print "Bad Handler: " + appfile.name
if handler != None:
navbody += handler(contents.content, contents.tbl_info,
contents.knowninfo)
navbody += "</div>"
navhead += "</ul></div>" #ul and panel-heading
navbody += "</div></div>"
html += """<div class="panel with-nav-tabs panel-default">
""" + navhead + navbody + """</div>"""
#do the nav-tabs init here and add items
else:
handler = None
try:
handler = self.handlers[appfile.ftype]
except:
#print "No Handler: " + appfile.name
html += "Error No Handler Implemented"
if handler != None:
html += handler(appfile.content, appfile.tbl_info)
#html = html.encode("UTF-8")
return html
def dump_hex(self, appfile):
if appfile.ftype == ExtractStore.TYPE_MULTI:
if appfile.filepath == None:
return "<p>Unexpected Error: no filepath specified</p>"
try:
f = open(appfile.filepath, "rb")
content = f.read()
except:
return """
<p>File {} not found. Please make sure that you haven't
unmounted any images regarding this case
""".format(cgi.escape(appfile.filepath))
return self._get_datahtml(content)
else:
return self._get_datahtml(appfile.content)
def dump_strings(self, appfile):
if appfile.ftype == ExtractStore.TYPE_MULTI:
if appfile.filepath == None:
return "<p>Unexpected Error: no filepath specified</p>"
try:
f = open(appfile.filepath, "rb")
content = f.read()
except:
return """
<p>File {} not found. Please make sure that you haven't
unmounted any images regarding this case
""".format(cgi.escape(appfile.filepath))
return self._get_stringshtml(self.restringsdump(content))
else:
return self._get_stringshtml(self.restringsdump(appfile.content))
def datahandler(self, value):
return """
<button class="btn btn-primary btn-sm" onclick="hideshow(this);">
Show Data <span class="glyphicon glyphicon-chevron-down"></span>
</button>
<pre style="display:none;">{hexdump}</pre>
""".format(hexdump=self.hexdump(value))
def datehandler(self, value):
return """<span class="date">{value}</span>""".format(value=value)
def texthandler(self, value):
print "TEXTHANDLER CALLED"
return "Unimplemented"
def hexdump(self, src, length=16):
if src == None:
return "None"
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.'
for x in range(256)])
lines = []
for c in xrange(0, len(src), length):
chars = src[c:c + length]
mhex = ' '.join(["%02x" % ord(x) for x in chars])
printable = cgi.escape(''.join([
"%s" % ((ord(x) <= 127 and FILTER[ord(x)]) or '.')
for x in chars
]))
lines.append("%04x %-*s %s\n" % (c, length * 3, mhex, printable))
return ''.join(lines)
def restringsdump(self, content):
for match in re.finditer("[^\x00-\x1F\x7F-\xFF]{4,}", content):
yield (match.start(), cgi.escape(match.group()))
def stringsdump(self, content, minlen=4):
result = ""
loc = 0
for c in content:
loc += 1
if c in string.printable:
result += c
continue
elif len(result) >= minlen:
yield (loc, cgi.escape(result))
result = ""
def _parse_column(self,
colname,
value,
tblinfo=None,
knowntable=None,
notd=False):
ret = u""
if isinstance(value, StringIO.StringIO):
value = value.buf
if notd:
valwrap = u"{value}"
else:
valwrap = u"<td>{value}</td>"
knownfields = None
if knowntable != None:
knownfields = knowntable.knownfields
if tblinfo != None:
if colname in tblinfo:
if tblinfo[colname] == "BLOB":
if value == None:
return valwrap.format(value="None")
mimetype = self.mime.get_buffermimetype(str(value), True)
if mimetype != None:
if mimetype.startswith("image/"):
return valwrap.format(value=self._get_imagehtml(
value, {"mime": mimetype}))
elif mimetype.startswith("bplist"):
return valwrap.format(
value=self._get_bplisthtml(value))
return valwrap.format(value=self.datahandler(value))
try:
if knownfields != None:
if colname in knownfields:
return valwrap.format(
value=self.datahandlers[knownfields[colname]](value))
return valwrap.format(value=value)
except Exception, e:
if type(value) == StringIO:
value = value.buf
print traceback.format_exc()
print "colname: {colname}".format(colname=colname)
sys.stderr.write(repr(e) + "\n")
# <button type="button" class="btn btn-primary btn-sm">Small button</button>
return valwrap.format(value=self.datahandler(value))
return ret
class ApplicationParser(object):
parser_name = "Android Application Parser"
appstore = {}
exstore = None
def __init__(self, outqueue=None, extract_store=None, mountpoint=None,
settings=None, versions=None ):
self.mimguess = MimeGuesser()
self.outqueue = outqueue
self.settings = settings
self.versions = versions
imp = Importer()
app_modules = imp.get_package_modules("AndroidApps", IApp())
for app in app_modules:
name = app.get_packagename()
if name in self.appstore:
self.appstore[name].append(app)
else:
self.appstore[name] = [app]
if extract_store != None:
self.exstore = extract_store
else:
self.exstore = ExtractStore.ExtractStore()
def get_extractedstore(self):
return self.exstore
def get_package_list(self):
return self.appstore.keys()
def get_package(self, name):
if name not in self.appstore:
return None
if len(self.appstore[name]) == 1:
if self.appstore[name][0].has_defaultversion() == False:
self.outqueue.put("Package {} is not default".format(name))
return self.appstore[name][0]
"""
If multiple entries for the same package exist, then we need to decide
which one we should use
"""
default = None
appversion = self.versions.get_application_version(name)
if appversion == None or appversion == "N/A":
self.outqueue.put("{} could not determine App version".format(name))
else:
try:
appversion = int(appversion)
except:
pass
for defn in self.appstore[name]:
if defn.has_defaultversion:
if default != None:
self.outqueue.put("{} has more than one default".format(name))
default = defn
if defn.has_version(appversion):
return defn
return default
def handle_file(self, filename, dirpath, app):
filepath = join(dirpath,filename)
handler = self.mimguess.get_handler(filepath)
knowninfo = app.get_file_info(filename)
fobject = None
if handler != None:
fobject = handler(filename,filepath,knowninfo,self.outqueue,
self.settings)
else:
"""
This is a remote case where no handler can handle the file.
However, the default handler for unknown files should always be the
raw data handler
"""
fobject = ExtractStore.ApplicationFile(filename,
ExtractStore.TYPE_NONE)
fobject.set_mime(self.mimguess.get_filemime(filepath))
fobject.add_sig(genfilesig(filepath, DEFAULT_HASHER))
fobject.add_stats(os.stat(filepath))
return fobject
def scan_single_app_dir(self, dirroot, app, dirobject, store_app):
for f in listdir(dirroot):
filepath = join(dirroot,f)
if isdir(filepath):
subdirobject = dirobject.add_dir(f)
self.scan_single_app_dir(filepath, app, subdirobject, store_app)
elif islink(filepath):
"""Skip for now"""
continue
elif isfile(filepath):
store_app.totalfiles += 1
dirobject.add_file(self.handle_file(f, dirroot, app))
return
def read_link(self, linkpath):
target_path = os.readlink(linkpath)
if os.path.isabs(target_path):
return target_path
return join(dirname(linkpath),target_path)
def scan_single_app(self, approot_path, app):
#print "[APP]: " + app.get_packagename()
if self.outqueue != None:
self.outqueue.put("[APP]: " + app.get_packagename())
store_app = self.exstore.create_application(
app.get_packagename(), app.get_canonicalname())
rootdir = store_app.add_directory(approot_path)
store_app.add_root_stats( os.stat(approot_path) )
for f in listdir(approot_path):
filepath = join(approot_path,f)
if isdir(filepath):
dirobject = rootdir.add_dir(f)
self.scan_single_app_dir(filepath, app, dirobject, store_app)
elif islink(filepath):
if filepath.endswith("lib"):
store_app.set_library_path(self.read_link(filepath))
elif isfile(filepath):
store_app.totalfiles += 1
rootdir.add_file(self.handle_file(f,approot_path,app))
return
def scan_directory(self, root):
for dfile in os.listdir(root):
fqname = join(root,dfile)
if isdir( fqname ):
app = self.get_package( dfile )
if app == None:
app = IApp()
app.set_packagename( dfile )
self.scan_single_app( fqname, app )
else:
print "Stray file found: " + fqname
#self.handle_file(fqname, None)
if self.outqueue != None:
self.outqueue.put("Scanned Total Apps: " + str(len(self.exstore.store)) )