def test_getInHMS(self):
print ">>> test_getInHMS"
app = LinkCrawler()
app.LOG_LEVEL = 10
timeInHMS = Iprods_Date_Utils.getInHMS(3600)
print "<<< test_getInHMS [seconds: 3600, timeInHMS: %s]\n" % (timeInHMS)
self.failUnless(timeInHMS)
def test_getInHMS(self):
print ">>> test_getInHMS"
app = LinkCrawler()
app.LOG_LEVEL = 10
timeInHMS = Iprods_Date_Utils.getInHMS(3600)
print "<<< test_getInHMS [seconds: 3600, timeInHMS: %s]\n" % (
timeInHMS)
self.failUnless(timeInHMS)
def generateReport(self, type):
self.log.debug(">>> generateReport [type: %s]", type)
data = cStringIO.StringIO()
if type == 'text':
data.write("Date: %s" % strftime("%Y-%m-%d %H:%M:%S"))
data.write("\nDomain: %s" % self.DOMAIN)
data.write("\nPath: %s" % self.PATH)
data.write("\nShow Statuscodes: %s" % self.REPORT_LEVEL)
if (self.EXCLUDE_PATTERN):
data.write("\nExclude pattern: %s" % self.EXCLUDE_PATTERN)
data.write("\nElapsed time: %s (h:m:s)" % Iprods_Date_Utils.getInHMS(self.elapsedTime))
data.write("\nResult [internal: %s, external: %s, excluded: %s, double: %s]" % (len(self.processedIntern), len(self.processedExtern), self.countExcluded, self.countDouble))
data.write("\n\nInternal Failed:")
if len(self.failedIntern) > 0:
for internal in self.failedIntern:
data.write("\n%s" % internal)
data.write("\n\nExternal Failed:")
if len(self.statusCodes) > 0:
#{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
#for key in sorted(self.statusCodes):
codes = self.REPORT_LEVEL.split(',')
for code in sorted(self.statusCodes, reverse=True):
Iprods_Http_Utils.isCodeInCodeRange(codes, code)
if self.showStatusCode(code):
data.write("\n\nHTTP_STATUS_CODE: %s\n" % code)
urls = self.statusCodes[code]
for url in urls:
data.write("\nURL: %s\n" % url)
for external in urls[url]:
data.write("EXT: %s\n" % external)
if type == 'csv':
if len(self.statusCodes) > 0:
data.write("HTTP_STATUS_CODE, URL, EXTERN")
#{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
for code in sorted(self.statusCodes, reverse=True):
if self.showStatusCode(code):
urls = self.statusCodes[code]
for url in urls:
for external in urls[url]:
data.write("\n%s, %s, %s" % (code, url, external))
#if len(self.processedExtern) > 0:
#p.write("\n\nProcessed internal links\n %s" % str(self.processedIntern))
# data.write("\n\nProcessed external links\n%s" % str(self.processedExtern))
report = data.getvalue()
data.close()
self.log.debug("<<< generateReport [report: %s]", report)
return report
def generateReport(self, type):
self.log.debug(">>> generateReport [type: %s]", type)
data = cStringIO.StringIO()
if type == 'text':
data.write("Date: %s" % strftime("%Y-%m-%d %H:%M:%S"))
data.write("\nDomain: %s" % self.DOMAIN)
data.write("\nPath: %s" % self.PATH)
data.write("\nShow Statuscodes: %s" % self.REPORT_LEVEL)
if (self.EXCLUDE_PATTERN):
data.write("\nExclude pattern: %s" % self.EXCLUDE_PATTERN)
data.write("\nElapsed time: %s (h:m:s)" %
Iprods_Date_Utils.getInHMS(self.elapsedTime))
data.write(
"\nResult [internal: %s, external: %s, excluded: %s, double: %s]"
% (len(self.processedIntern), len(self.processedExtern),
self.countExcluded, self.countDouble))
data.write("\n\nInternal Failed:")
if len(self.failedIntern) > 0:
for internal in self.failedIntern:
data.write("\n%s" % internal)
data.write("\n\nExternal Failed:")
if len(self.statusCodes) > 0:
#{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
#for key in sorted(self.statusCodes):
codes = self.REPORT_LEVEL.split(',')
for code in sorted(self.statusCodes, reverse=True):
Iprods_Http_Utils.isCodeInCodeRange(codes, code)
if self.showStatusCode(code):
data.write("\n\nHTTP_STATUS_CODE: %s\n" % code)
urls = self.statusCodes[code]
for url in urls:
data.write("\nURL: %s\n" % url)
for external in urls[url]:
data.write("EXT: %s\n" % external)
if type == 'csv':
if len(self.statusCodes) > 0:
data.write("HTTP_STATUS_CODE, URL, EXTERN")
#{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
for code in sorted(self.statusCodes, reverse=True):
if self.showStatusCode(code):
urls = self.statusCodes[code]
for url in urls:
for external in urls[url]:
data.write("\n%s, %s, %s" %
(code, url, external))
#if len(self.processedExtern) > 0:
#p.write("\n\nProcessed internal links\n %s" % str(self.processedIntern))
# data.write("\n\nProcessed external links\n%s" % str(self.processedExtern))
report = data.getvalue()
data.close()
self.log.debug("<<< generateReport [report: %s]", report)
return report
