def TestTime():
db = SqliteDatabase(Constants.NSRLDBName)
if not db.OpenConnection():
return
newDB = SqliteDatabase("NewNSRL.db")
if not newDB.OpenConnection():
return
md5 = "8BA8BC04896C421A704282E9B87B5520"
query = "select MD5 from " + Constants.NSRLFileTable + " where MD5= '%s' ;" % md5
start = time.time()
rows = db.FetchAllRows(query)
if len(rows) > 0:
print 'found'
else:
print 'not found'
end = time.time()
print 'elapsed time ', end - start
start = time.time()
bucketID = CommonFunctions.GetMD5HashBucketID(md5)
query = "select MD5 from a%d where MD5='%s';" % (bucketID, md5)
rows = newDB.FetchAllRows(query)
if len(rows) > 0:
print 'found'
else:
print 'not found'
end = time.time()
print 'bucket elapsed time ', end - start
def CheckMD5Hash(self, db, MD5Hash):
tableName = CommonFunctions.GetMD5HashBucketID(MD5Hash)
# = 'a%d'%bucketID
query = "select MD5 from %s where MD5 = '%s';"%(tableName, MD5Hash)
rows = db.FetchOneRow(query)
if rows:
return True
else:
return False
def ImportNSRLHashAndProduct(self):
#q = Queue.Queue()
db = SqliteDatabase(Constants.NSRLDBName)
if not db.OpenConnection():
return
Hashes = set()
#txtFile = False
#query = "INSERT INTO " + Constants.NSRLFileTable + " (MD5, FileName, ProductCode, OSCode) values (?,?,?,?)"
try:
fin = open(os.path.join(self.dirPath, "NSRLFile.txt"))
#fin = gzip.open(self.dirPath + PlatformMethods.GetDirSeparator() + "NSRLFile.txt.gz")
except IOError:
#fin = open(self.dirPath + PlatformMethods.GetDirSeparator() + "NSRLFile.txt")
#txtFile = True
print 'Error Opening file: ', os.path.join(self.dirPath,
"NSRLFile.txt")
return
#pass
# Get the file size:y
try:
fin.seek(0, 2)
size = fin.tell()
fin.seek(0)
except TypeError:
size = None
#finCSV = csv.reader(fin)
#print "Startime = %s"%time.asctime()
#i = 0
self.EstimatedTime = ""
#while self.keepGoing:
count = 0
self.startTime = time.time()
#manyValues = []
#totalRows = len(finCSV)
skip = True
"""
t = Thread(target=worker, args=(q,))
t.setDaemon(True)
t.start()
"""
while fin:
rows = fin.readlines(10000)
if not rows:
break
#for row in finCSV:
for row in rows:
if not self.keepGoing:
break
if skip:
skip = False
continue
md5 = row.split(',')[1].replace('"', '')
table = CommonFunctions.GetMD5HashBucketID(md5)
#print col
#return
#q.put([table, md5])
#md5 = row[1]
if table not in Hashes:
Hashes.add(table)
db.ExecuteNonQuery(
"CREATE TABLE IF NOT EXISTS %s (MD5 varchar(32) primary key);"
% table)
#db.ExecuteNonQuery("CREATE INDEX Ind%s ON table (MD5);"%table)
try:
db.ExecuteNonQuery("insert into %s (MD5) values ('%s');" %
(table, md5))
except Exception, value:
#print 'Error :: ', value
pass
if size and not count % 10000:
done = fin.tell()
self.gaugeValue = float(done * 100) / float(size)
#print "Progress %02u%% Done - %uk rows\r" % (done*100/size,count/1000)
self.ElapsedTime = time.time()
#self.gaugeValue = (float(size)/float(driveSize))*100
timeTaken = float(self.ElapsedTime - self.startTime)
if timeTaken == 0:
timeTaken = 1
rate = float(done) / timeTaken
self.rateInfo = "%.2fMB of %.2fMB at %.2fMB/sec" % (
done / 1024. / 1024, size / 1024. / 1024,
rate / 1024. / 1024)
self.Status = "Updating NSRL Software Hashes... [%.2f" % self.gaugeValue
self.Status += "%]"
#self.startTime = time.time()
self.SendEvent()
if count == 10000:
self.EstimatedTime = CommonFunctions.ConvertSecondsToDayHourMinSec(
float(size) / rate)
count += 1
