def test_getInHMS(self):
        print ">>> test_getInHMS"
        app = LinkCrawler()
        app.LOG_LEVEL = 10
        timeInHMS = Iprods_Date_Utils.getInHMS(3600)

        print "<<< test_getInHMS [seconds: 3600, timeInHMS: %s]\n" % (timeInHMS)
        self.failUnless(timeInHMS)
Example #2
0
    def test_getInHMS(self):
        print ">>> test_getInHMS"
        app = LinkCrawler()
        app.LOG_LEVEL = 10
        timeInHMS = Iprods_Date_Utils.getInHMS(3600)

        print "<<< test_getInHMS [seconds: 3600, timeInHMS: %s]\n" % (
            timeInHMS)
        self.failUnless(timeInHMS)
Example #3
0
    def generateReport(self, type):
        self.log.debug(">>> generateReport [type: %s]", type)
        data = cStringIO.StringIO()

        if type == 'text':
            data.write("Date: %s" % strftime("%Y-%m-%d %H:%M:%S"))
            data.write("\nDomain: %s" % self.DOMAIN)
            data.write("\nPath: %s" % self.PATH)
            data.write("\nShow Statuscodes: %s" % self.REPORT_LEVEL)
            if (self.EXCLUDE_PATTERN):
                data.write("\nExclude pattern: %s" % self.EXCLUDE_PATTERN)
            data.write("\nElapsed time: %s (h:m:s)" % Iprods_Date_Utils.getInHMS(self.elapsedTime))
            data.write("\nResult [internal: %s, external: %s, excluded: %s, double: %s]" % (len(self.processedIntern), len(self.processedExtern), self.countExcluded, self.countDouble))

            data.write("\n\nInternal Failed:")

            if len(self.failedIntern) > 0:
                for internal in self.failedIntern:
                    data.write("\n%s" % internal)

            data.write("\n\nExternal Failed:")

            if len(self.statusCodes) > 0:
                #{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
                #for key in sorted(self.statusCodes):
                codes = self.REPORT_LEVEL.split(',')
                for code in sorted(self.statusCodes, reverse=True):

                    Iprods_Http_Utils.isCodeInCodeRange(codes, code)
                    if self.showStatusCode(code):
                        data.write("\n\nHTTP_STATUS_CODE: %s\n" % code)
                        urls = self.statusCodes[code]
                        for url in urls:
                            data.write("\nURL: %s\n" % url)
                            for external in urls[url]:
                                data.write("EXT: %s\n" % external)


        if type == 'csv':
            if len(self.statusCodes) > 0:
                data.write("HTTP_STATUS_CODE, URL, EXTERN")
                #{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
                for code in sorted(self.statusCodes, reverse=True):
                    if self.showStatusCode(code):
                        urls = self.statusCodes[code]
                        for url in urls:
                            for external in urls[url]:
                                data.write("\n%s, %s, %s" % (code, url, external))

        #if len(self.processedExtern) > 0:
            #p.write("\n\nProcessed internal links\n %s" % str(self.processedIntern))
        #	data.write("\n\nProcessed external links\n%s" % str(self.processedExtern))

        report = data.getvalue()
        data.close()
        self.log.debug("<<< generateReport [report: %s]", report)
        return report
Example #4
0
    def generateReport(self, type):
        self.log.debug(">>> generateReport [type: %s]", type)
        data = cStringIO.StringIO()

        if type == 'text':
            data.write("Date: %s" % strftime("%Y-%m-%d %H:%M:%S"))
            data.write("\nDomain: %s" % self.DOMAIN)
            data.write("\nPath: %s" % self.PATH)
            data.write("\nShow Statuscodes: %s" % self.REPORT_LEVEL)
            if (self.EXCLUDE_PATTERN):
                data.write("\nExclude pattern: %s" % self.EXCLUDE_PATTERN)
            data.write("\nElapsed time: %s (h:m:s)" %
                       Iprods_Date_Utils.getInHMS(self.elapsedTime))
            data.write(
                "\nResult [internal: %s, external: %s, excluded: %s, double: %s]"
                % (len(self.processedIntern), len(self.processedExtern),
                   self.countExcluded, self.countDouble))

            data.write("\n\nInternal Failed:")

            if len(self.failedIntern) > 0:
                for internal in self.failedIntern:
                    data.write("\n%s" % internal)

            data.write("\n\nExternal Failed:")

            if len(self.statusCodes) > 0:
                #{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
                #for key in sorted(self.statusCodes):
                codes = self.REPORT_LEVEL.split(',')
                for code in sorted(self.statusCodes, reverse=True):

                    Iprods_Http_Utils.isCodeInCodeRange(codes, code)
                    if self.showStatusCode(code):
                        data.write("\n\nHTTP_STATUS_CODE: %s\n" % code)
                        urls = self.statusCodes[code]
                        for url in urls:
                            data.write("\nURL: %s\n" % url)
                            for external in urls[url]:
                                data.write("EXT: %s\n" % external)

        if type == 'csv':
            if len(self.statusCodes) > 0:
                data.write("HTTP_STATUS_CODE, URL, EXTERN")
                #{'200' : { 'URL' : [EXTERNALLINK1, EXTERNALLINK2 ....] }, '404' : {'URL' : [....]}
                for code in sorted(self.statusCodes, reverse=True):
                    if self.showStatusCode(code):
                        urls = self.statusCodes[code]
                        for url in urls:
                            for external in urls[url]:
                                data.write("\n%s, %s, %s" %
                                           (code, url, external))

        #if len(self.processedExtern) > 0:
        #p.write("\n\nProcessed internal links\n %s" % str(self.processedIntern))
        #	data.write("\n\nProcessed external links\n%s" % str(self.processedExtern))

        report = data.getvalue()
        data.close()
        self.log.debug("<<< generateReport [report: %s]", report)
        return report