def run(self):
while True:
domain = HTTPSQSQueue.get(DOMAINQUEUE02).lower()
if '' == domain and None == time.sleep(1): continue
if self.isDomainInBlacklist(domain):
C.Info('Domain in black list: %s' % domain, C.DEBUG)
continue
self.f**k(domain)
self.refreshBlacklist(domain)
self.monitor()
def checkDomainsQueue(self):
self._QueueUnRead01 = self.getQueueUnRead(DOMAINQUEUE01)
self._QueueUnRead02 = self.getQueueUnRead(DOMAINQUEUE02)
if self._QueueUnRead01 > 10000:
C.Info('HTTPSQSQueue %s will be full, waiting for cache!!!!!!!!!!!!!' % DOMAINQUEUE01, C.ALERT)
self.cacheHTTPSQSQueue(DOMAINQUEUE01)
C.Info('HTTPSQSQueue cached', C.ALERT)
if self._QueueUnRead02 > 10000:
C.Info('HTTPSQSQueue %s will be full, waiting for reset!!!!!!!!!!!!!' % DOMAINQUEUE02, C.ALERT)
for i in range(0, 100):HTTPSQSQueue.put(DOMAINQUEUE01, HTTPSQSQueue.get(DOMAINQUEUE02))
HTTPSQSQueue.reset(DOMAINQUEUE02)
def f**k(self):
while True:
originalDomain = HTTPSQSQueue.get(DOMAINQUEUE01)
if originalDomain == '':
time.sleep(5)
continue
self.fuckDomain(originalDomain)
self.ThreadRefreshTime = time.time()
self.TotalProcessed = self.TotalProcessed + 1
info = '*****(%2d) processed %d domains in %.fs' % (self._tid, self.TotalProcessed, self.ThreadRefreshTime-self.ThreadStartTime)
C.Info(info, C.DEBUG)
if self.ThreadCanExit == True: break
def cacheHTTPSQSQueue(self,qName):
while self.getQueueUnRead(qName) > 1000:
domains = []
for i in range(0, 100):
domains.append(HTTPSQSQueue.get(DOMAINQUEUE01))
try:
cacheFileName = qName + '-' + time.strftime('%Y%m%d%H%M%S') + '.qc'
f = open(cacheFileName, 'a+')
try:
f.write('\n'.join(domains))
except:
f.close()
except:
pass
