def test_write2file(self):
print ">>> test_write2file"
writeResult = True
file = ""
try:
app = LinkCrawler()
file = "%s%s.txt" % (app.REPORT_PATH, Iprods_File_Utils.getValidFileName("unittest"))
Iprods_File_Utils.write2file(file, "Webreport Unittest test_write2file")
except:
writeResult = False
exctype, value = sys.exc_info()[:2]
print("exctype: %s, value: %s" % (exctype, value))
print "<<< test_write2file [file: %s]\n" % (file)
self.failUnless(writeResult)
def test_write2file(self):
print ">>> test_write2file"
writeResult = True
file = ""
try:
app = LinkCrawler()
file = "%s%s.txt" % (
app.REPORT_PATH,
Iprods_File_Utils.getValidFileName("unittest"))
Iprods_File_Utils.write2file(file,
"Webreport Unittest test_write2file")
except:
writeResult = False
exctype, value = sys.exc_info()[:2]
print("exctype: %s, value: %s" % (exctype, value))
print "<<< test_write2file [file: %s]\n" % (file)
self.failUnless(writeResult)
def run(self):
self.log.debug(">>> run [DOMAIN: %s, path: %s]" %
(self.DOMAIN, self.PATH))
try:
start = time.time()
printTime = start + 30
isFinished = False
reader_threads = [Reader(self) for i in range(self.MAX_THREADS)]
for thread in reader_threads:
thread.setDaemon(True)
thread.start()
if self.CHECK_EXTERNAL is True:
checker_threads = [
Checker(self) for i in range(self.MAX_THREADS)
]
for thread in checker_threads:
thread.setDaemon(True)
thread.start()
Reader.lock.acquire()
Reader.result[self.PATH] = "busy"
Reader.lock.release()
Reader.queue.put(self.PATH)
while True:
actTime = time.time()
lastTime = actTime - self.lastReader
#either queues are empty or reached max idle time
if round(printTime, 0) <= round(actTime, 0):
Reader.lock.acquire()
lenReader = len(Reader.result)
Reader.lock.release()
Checker.lock.acquire()
lenChecker = len(Checker.result)
Checker.lock.release()
print "status [links-read: %s, external-links: %s]" % (
lenReader, lenChecker)
printTime = actTime + 15
if (lastTime > 30)\
and Reader.queue.empty() and (self.CHECK_EXTERNAL is False or Checker.queue.empty()):
#print "status [readerIsEmpty: %s, checkerIsEmpty: %s, lastReader: %s, lastTime: %s]" % (Reader.queue.empty(), Checker.queue.empty(), self.lastReader, lastTime)
break
Checker.queue.join()
self.elapsedTime = (time.time() - start)
if self.VERBOSE_REPORT or self.WRITE_REPORT or self.SEND_REPORT:
report = self.generateReport('text')
csv = self.generateReport('csv')
logFilename = Iprods_File_Utils.getValidFileName(
self.DOMAIN, True)
txtFile = "%s%s.txt" % (self.REPORT_PATH, logFilename)
csvFile = "%s%s.csv" % (self.REPORT_PATH, logFilename)
if self.VERBOSE_REPORT:
print report
if self.WRITE_REPORT:
try:
Iprods_File_Utils.write2file(txtFile, report)
except:
exctype, value = sys.exc_info()[:2]
self.log.exception(
"write2file failed. exctype: %s, value: %s [filename: %s, report: %s]"
% (exctype, value, txtFile, report))
try:
Iprods_File_Utils.write2file(csvFile, csv)
except:
exctype, value = sys.exc_info()[:2]
self.log.exception(
"write2file failed. exctype: %s, value: %s [filename: %s, csv: %s]"
% (exctype, value, csvFile, csv))
if self.SEND_REPORT:
self.smtpsend(self.MAIL_FROM, self.MAIL_CC, self.MAIL_BCC,\
self.MAIL_TO, self.MAIL_SUBJECT, report, csvFile)
except StandardError:
exctype, value = sys.exc_info()[:2]
self.log.exception("run failed exc: %s, val: %s" %
(exctype, value))
self.log.debug("<<< run")
def run(self):
self.log.debug(">>> run [DOMAIN: %s, path: %s]" % (self.DOMAIN, self.PATH))
try:
start = time.time()
printTime = start + 30
isFinished = False
reader_threads = [Reader(self) for i in range(self.MAX_THREADS)]
for thread in reader_threads:
thread.setDaemon(True)
thread.start()
if self.CHECK_EXTERNAL is True:
checker_threads = [Checker(self) for i in range(self.MAX_THREADS)]
for thread in checker_threads:
thread.setDaemon(True)
thread.start()
Reader.lock.acquire()
Reader.result[self.PATH] = "busy"
Reader.lock.release()
Reader.queue.put(self.PATH)
while True:
actTime = time.time()
lastTime = actTime - self.lastReader
#either queues are empty or reached max idle time
if round(printTime,0) <= round(actTime,0):
Reader.lock.acquire()
lenReader = len(Reader.result)
Reader.lock.release()
Checker.lock.acquire()
lenChecker = len(Checker.result)
Checker.lock.release()
print "status [links-read: %s, external-links: %s]" % (lenReader, lenChecker)
printTime = actTime + 15
if (lastTime > 30)\
and Reader.queue.empty() and (self.CHECK_EXTERNAL is False or Checker.queue.empty()):
#print "status [readerIsEmpty: %s, checkerIsEmpty: %s, lastReader: %s, lastTime: %s]" % (Reader.queue.empty(), Checker.queue.empty(), self.lastReader, lastTime)
break;
Checker.queue.join()
self.elapsedTime = (time.time() - start)
if self.VERBOSE_REPORT or self.WRITE_REPORT or self.SEND_REPORT:
report = self.generateReport('text')
csv = self.generateReport('csv')
logFilename = Iprods_File_Utils.getValidFileName(self.DOMAIN, True)
txtFile = "%s%s.txt" % (self.REPORT_PATH, logFilename)
csvFile = "%s%s.csv" % (self.REPORT_PATH, logFilename)
if self.VERBOSE_REPORT:
print report
if self.WRITE_REPORT:
try:
Iprods_File_Utils.write2file(txtFile, report)
except:
exctype, value = sys.exc_info()[:2]
self.log.exception("write2file failed. exctype: %s, value: %s [filename: %s, report: %s]" % (exctype, value, txtFile, report))
try:
Iprods_File_Utils.write2file(csvFile, csv)
except:
exctype, value = sys.exc_info()[:2]
self.log.exception("write2file failed. exctype: %s, value: %s [filename: %s, csv: %s]" % (exctype, value, csvFile, csv))
if self.SEND_REPORT:
self.smtpsend(self.MAIL_FROM, self.MAIL_CC, self.MAIL_BCC,\
self.MAIL_TO, self.MAIL_SUBJECT, report, csvFile)
except StandardError:
exctype, value = sys.exc_info()[:2]
self.log.exception("run failed exc: %s, val: %s" % (exctype, value))
self.log.debug("<<< run")
