def parse_sqlite( filename, filepath, known_info, print_queue, settings=None ): tmpdir = tempfile.gettempdir() tmp_filepath = os.path.join( tmpdir, filename ) print_queue.put( "\t[SQLite3 Parsing] Copying file: {} > {}".format( filepath, tmp_filepath ) ) shutil.copy2( filepath, tmp_filepath ) conn = sqlite3.connect(tmp_filepath) SQLite3Initializer.init_android(conn) conn.row_factory = dict_factory cur = conn.cursor() subsections = [] for descr, known_fields in known_info.knownfields.iteritems(): items = _parse_sqlite_knownfield( known_fields, cur, print_queue ) subsection = ExtractStore.MiscSubSection( descr, items ) subsections.append( subsection ) return subsections
def handler_sqlite(filename, filepath, info, out=None, settings=None ): if out != None: out.put("\t[SQLite] Extracting: " + filepath) else: print "\t[SQLite] Extracting: " + filepath fobj = ExtractStore.ApplicationFile(filename,ExtractStore.TYPE_MULTI, filepath=filepath) #establish connection if settings != None: tempfile.tempdir = settings.get("tempdir") tmpdir = tempfile.gettempdir() tmp_filepath = os.path.join(tmpdir,filename) if out != None: out.put("\t\t-> %s" % tmp_filepath) else: print "\t\t-> %s" % tmp_filepath shutil.copy2( filepath, tmp_filepath) conn = sqlite3.connect(tmp_filepath) conn.row_factory = dict_factory SQLite3Initializer.init_android(conn) cur = conn.cursor() #pull schema note that we also need to parse VIEWs but we need to define #collations and functions... schemaq = """ SELECT name,type,sql FROM sqlite_master WHERE sql NOT NULL AND type == 'table' OR type == 'view' """ cur.execute(schemaq) schema = cur.fetchall() knowntable = None for tblrow in schema: name = tblrow["name"] type = tblrow["type"] sql = tblrow["sql"] if name.startswith("sqlite_"): continue try: cur.execute("PRAGMA table_info('{name}')".format(name=name)) except: continue columns = cur.fetchall() columns = [ col for col in columns if ":" not in col["name"] ] #columns = [{'name':str(c[1]),'type':str(c[2])} for c in fetch] if info != None: knowntable = get_known_table(info, name) if knowntable != None and knowntable.sql != None: cur.execute(knowntable.sql) res = cur.fetchall() else: if len(columns) != 0: q = "SELECT %s FROM %s" % (",".join( [c["name"] for c in columns] ), name ) #print "Executing: " + q try: cur.execute(q) except sqlite3.OperationalError as e: out.put("[Error]: {}\n\tSQL: {}\n\tSkipping Table".format( e.message,q)) continue except: return handle_data(filename, filepath, info) res = cur.fetchall() else: res = [] if knowntable != None: if knowntable.converter != None and len(knowntable.converter) != 0: for row in res: for conv in knowntable.converter.keys(): try: row[conv] = knowntable.converter[conv](row[conv]) except: out.put( "**[ERROR]** Invalid Column Name: {}".format(conv)) fobj.add_content(res, ExtractStore.TYPE_TABLE, name, columns, knowntable) #got all the data for this table + converted them cur.close() conn.close() os.remove(tmp_filepath) return fobj