def TestTime(): db = SqliteDatabase(Constants.NSRLDBName) if not db.OpenConnection(): return newDB = SqliteDatabase("NewNSRL.db") if not newDB.OpenConnection(): return md5 = "8BA8BC04896C421A704282E9B87B5520" query = "select MD5 from " + Constants.NSRLFileTable + " where MD5= '%s' ;" % md5 start = time.time() rows = db.FetchAllRows(query) if len(rows) > 0: print 'found' else: print 'not found' end = time.time() print 'elapsed time ', end - start start = time.time() bucketID = CommonFunctions.GetMD5HashBucketID(md5) query = "select MD5 from a%d where MD5='%s';" % (bucketID, md5) rows = newDB.FetchAllRows(query) if len(rows) > 0: print 'found' else: print 'not found' end = time.time() print 'bucket elapsed time ', end - start
def CheckMD5Hash(self, db, MD5Hash): tableName = CommonFunctions.GetMD5HashBucketID(MD5Hash) # = 'a%d'%bucketID query = "select MD5 from %s where MD5 = '%s';"%(tableName, MD5Hash) rows = db.FetchOneRow(query) if rows: return True else: return False
def ImportNSRLHashAndProduct(self): #q = Queue.Queue() db = SqliteDatabase(Constants.NSRLDBName) if not db.OpenConnection(): return Hashes = set() #txtFile = False #query = "INSERT INTO " + Constants.NSRLFileTable + " (MD5, FileName, ProductCode, OSCode) values (?,?,?,?)" try: fin = open(os.path.join(self.dirPath, "NSRLFile.txt")) #fin = gzip.open(self.dirPath + PlatformMethods.GetDirSeparator() + "NSRLFile.txt.gz") except IOError: #fin = open(self.dirPath + PlatformMethods.GetDirSeparator() + "NSRLFile.txt") #txtFile = True print 'Error Opening file: ', os.path.join(self.dirPath, "NSRLFile.txt") return #pass # Get the file size:y try: fin.seek(0, 2) size = fin.tell() fin.seek(0) except TypeError: size = None #finCSV = csv.reader(fin) #print "Startime = %s"%time.asctime() #i = 0 self.EstimatedTime = "" #while self.keepGoing: count = 0 self.startTime = time.time() #manyValues = [] #totalRows = len(finCSV) skip = True """ t = Thread(target=worker, args=(q,)) t.setDaemon(True) t.start() """ while fin: rows = fin.readlines(10000) if not rows: break #for row in finCSV: for row in rows: if not self.keepGoing: break if skip: skip = False continue md5 = row.split(',')[1].replace('"', '') table = CommonFunctions.GetMD5HashBucketID(md5) #print col #return #q.put([table, md5]) #md5 = row[1] if table not in Hashes: Hashes.add(table) db.ExecuteNonQuery( "CREATE TABLE IF NOT EXISTS %s (MD5 varchar(32) primary key);" % table) #db.ExecuteNonQuery("CREATE INDEX Ind%s ON table (MD5);"%table) try: db.ExecuteNonQuery("insert into %s (MD5) values ('%s');" % (table, md5)) except Exception, value: #print 'Error :: ', value pass if size and not count % 10000: done = fin.tell() self.gaugeValue = float(done * 100) / float(size) #print "Progress %02u%% Done - %uk rows\r" % (done*100/size,count/1000) self.ElapsedTime = time.time() #self.gaugeValue = (float(size)/float(driveSize))*100 timeTaken = float(self.ElapsedTime - self.startTime) if timeTaken == 0: timeTaken = 1 rate = float(done) / timeTaken self.rateInfo = "%.2fMB of %.2fMB at %.2fMB/sec" % ( done / 1024. / 1024, size / 1024. / 1024, rate / 1024. / 1024) self.Status = "Updating NSRL Software Hashes... [%.2f" % self.gaugeValue self.Status += "%]" #self.startTime = time.time() self.SendEvent() if count == 10000: self.EstimatedTime = CommonFunctions.ConvertSecondsToDayHourMinSec( float(size) / rate) count += 1