def test_write2file(self): print ">>> test_write2file" writeResult = True file = "" try: app = LinkCrawler() file = "%s%s.txt" % (app.REPORT_PATH, Iprods_File_Utils.getValidFileName("unittest")) Iprods_File_Utils.write2file(file, "Webreport Unittest test_write2file") except: writeResult = False exctype, value = sys.exc_info()[:2] print("exctype: %s, value: %s" % (exctype, value)) print "<<< test_write2file [file: %s]\n" % (file) self.failUnless(writeResult)
def test_write2file(self): print ">>> test_write2file" writeResult = True file = "" try: app = LinkCrawler() file = "%s%s.txt" % ( app.REPORT_PATH, Iprods_File_Utils.getValidFileName("unittest")) Iprods_File_Utils.write2file(file, "Webreport Unittest test_write2file") except: writeResult = False exctype, value = sys.exc_info()[:2] print("exctype: %s, value: %s" % (exctype, value)) print "<<< test_write2file [file: %s]\n" % (file) self.failUnless(writeResult)
def run(self): self.log.debug(">>> run [DOMAIN: %s, path: %s]" % (self.DOMAIN, self.PATH)) try: start = time.time() printTime = start + 30 isFinished = False reader_threads = [Reader(self) for i in range(self.MAX_THREADS)] for thread in reader_threads: thread.setDaemon(True) thread.start() if self.CHECK_EXTERNAL is True: checker_threads = [ Checker(self) for i in range(self.MAX_THREADS) ] for thread in checker_threads: thread.setDaemon(True) thread.start() Reader.lock.acquire() Reader.result[self.PATH] = "busy" Reader.lock.release() Reader.queue.put(self.PATH) while True: actTime = time.time() lastTime = actTime - self.lastReader #either queues are empty or reached max idle time if round(printTime, 0) <= round(actTime, 0): Reader.lock.acquire() lenReader = len(Reader.result) Reader.lock.release() Checker.lock.acquire() lenChecker = len(Checker.result) Checker.lock.release() print "status [links-read: %s, external-links: %s]" % ( lenReader, lenChecker) printTime = actTime + 15 if (lastTime > 30)\ and Reader.queue.empty() and (self.CHECK_EXTERNAL is False or Checker.queue.empty()): #print "status [readerIsEmpty: %s, checkerIsEmpty: %s, lastReader: %s, lastTime: %s]" % (Reader.queue.empty(), Checker.queue.empty(), self.lastReader, lastTime) break Checker.queue.join() self.elapsedTime = (time.time() - start) if self.VERBOSE_REPORT or self.WRITE_REPORT or self.SEND_REPORT: report = self.generateReport('text') csv = self.generateReport('csv') logFilename = Iprods_File_Utils.getValidFileName( self.DOMAIN, True) txtFile = "%s%s.txt" % (self.REPORT_PATH, logFilename) csvFile = "%s%s.csv" % (self.REPORT_PATH, logFilename) if self.VERBOSE_REPORT: print report if self.WRITE_REPORT: try: Iprods_File_Utils.write2file(txtFile, report) except: exctype, value = sys.exc_info()[:2] self.log.exception( "write2file failed. exctype: %s, value: %s [filename: %s, report: %s]" % (exctype, value, txtFile, report)) try: Iprods_File_Utils.write2file(csvFile, csv) except: exctype, value = sys.exc_info()[:2] self.log.exception( "write2file failed. exctype: %s, value: %s [filename: %s, csv: %s]" % (exctype, value, csvFile, csv)) if self.SEND_REPORT: self.smtpsend(self.MAIL_FROM, self.MAIL_CC, self.MAIL_BCC,\ self.MAIL_TO, self.MAIL_SUBJECT, report, csvFile) except StandardError: exctype, value = sys.exc_info()[:2] self.log.exception("run failed exc: %s, val: %s" % (exctype, value)) self.log.debug("<<< run")
def run(self): self.log.debug(">>> run [DOMAIN: %s, path: %s]" % (self.DOMAIN, self.PATH)) try: start = time.time() printTime = start + 30 isFinished = False reader_threads = [Reader(self) for i in range(self.MAX_THREADS)] for thread in reader_threads: thread.setDaemon(True) thread.start() if self.CHECK_EXTERNAL is True: checker_threads = [Checker(self) for i in range(self.MAX_THREADS)] for thread in checker_threads: thread.setDaemon(True) thread.start() Reader.lock.acquire() Reader.result[self.PATH] = "busy" Reader.lock.release() Reader.queue.put(self.PATH) while True: actTime = time.time() lastTime = actTime - self.lastReader #either queues are empty or reached max idle time if round(printTime,0) <= round(actTime,0): Reader.lock.acquire() lenReader = len(Reader.result) Reader.lock.release() Checker.lock.acquire() lenChecker = len(Checker.result) Checker.lock.release() print "status [links-read: %s, external-links: %s]" % (lenReader, lenChecker) printTime = actTime + 15 if (lastTime > 30)\ and Reader.queue.empty() and (self.CHECK_EXTERNAL is False or Checker.queue.empty()): #print "status [readerIsEmpty: %s, checkerIsEmpty: %s, lastReader: %s, lastTime: %s]" % (Reader.queue.empty(), Checker.queue.empty(), self.lastReader, lastTime) break; Checker.queue.join() self.elapsedTime = (time.time() - start) if self.VERBOSE_REPORT or self.WRITE_REPORT or self.SEND_REPORT: report = self.generateReport('text') csv = self.generateReport('csv') logFilename = Iprods_File_Utils.getValidFileName(self.DOMAIN, True) txtFile = "%s%s.txt" % (self.REPORT_PATH, logFilename) csvFile = "%s%s.csv" % (self.REPORT_PATH, logFilename) if self.VERBOSE_REPORT: print report if self.WRITE_REPORT: try: Iprods_File_Utils.write2file(txtFile, report) except: exctype, value = sys.exc_info()[:2] self.log.exception("write2file failed. exctype: %s, value: %s [filename: %s, report: %s]" % (exctype, value, txtFile, report)) try: Iprods_File_Utils.write2file(csvFile, csv) except: exctype, value = sys.exc_info()[:2] self.log.exception("write2file failed. exctype: %s, value: %s [filename: %s, csv: %s]" % (exctype, value, csvFile, csv)) if self.SEND_REPORT: self.smtpsend(self.MAIL_FROM, self.MAIL_CC, self.MAIL_BCC,\ self.MAIL_TO, self.MAIL_SUBJECT, report, csvFile) except StandardError: exctype, value = sys.exc_info()[:2] self.log.exception("run failed exc: %s, val: %s" % (exctype, value)) self.log.debug("<<< run")