Exemplo n.º 1
0
 def __init__(self, url, config=None):
     """
     Define a WebSite.
     """
     self.config = config
     self.webData = None
     self.crawl = False
     self.domainBase = None
     self.max_page_count = 1
     if config is not None:
         self.crawl = self.config.has_crawl()
         self.domainBase = self.config.get_domainbase()
         self.max_page_count = self.config.get_max_page_count()
     self.get_hrefs = KeeperReg("href\s*=\s*[^ <>]*[a-zA-Z0-9]")
     self.toProcess = [url]
     self.max_page_count -= 1
     self.processed = []
     self.iterate_pages()
Exemplo n.º 2
0
class WebSite(object):
    """
    Class representing a website.
    """
    def __init__(self, url, config=None):
        """
        Define a WebSite.
        """
        self.config = config
        self.webData = None
        self.crawl = False
        self.domainBase = None
        self.max_page_count = 1
        if config is not None:
            self.crawl = self.config.has_crawl()
            self.domainBase = self.config.get_domainbase()
            self.max_page_count = self.config.get_max_page_count()
        self.get_hrefs = KeeperReg("href\s*=\s*[^ <>]*[a-zA-Z0-9]")
        self.toProcess = [url]
        self.max_page_count -= 1
        self.processed = []
        self.iterate_pages()

    def get_next(self):
        """
        @return: The next URL in the list.
        """
        if len(self.toProcess) > 0:
            out = self.toProcess.pop()
            self.processed.append(out)
            return out
        return None

    def get_urls(self, data):
        """
        Process the page looking for URLs to traverse

        @param data: The raw webpage text.
        """
        if self.get_hrefs is not None and data is not None:
            for url in self.get_hrefs.keep(data):
                # This is removing junk from the string.  It needs to be
                # cleaner.
                #print "Start: %s" % url
                url = url.replace("href", "")
                url = url.replace("=", "")
                url = url.replace("\"", "")
                url = url.replace('\\','')
                url = url.strip()
                parsed = urlparse.urlparse(url)
                url = ("http" if len(parsed.scheme) == 0 else parsed.scheme)  + \
                    "://" + parsed.netloc + parsed.path
                if url not in self.toProcess and \
                   url not in self.processed and \
                   self.max_page_count > 0:
                    if self.domainBase is not None:
                        if url.rfind(self.domainBase) != -1 and \
                           url.rfind("@%s" % self.domainBase) == -1:
                            self.toProcess.append(url)
                            self.max_page_count -= 1
                    else:
                        self.toProcess.append(url)
                        self.max_page_count -= 1
                    

    def iterate_pages(self):
        """
        Iterate through the list of URLS available for use.
        """
        url = self.get_next()
        logger.debug("iterating over %s" % url)
        while url is not None:
            logger.debug("toProcess: %s" % self.toProcess)
            logger.debug("Processed: %s" % self.processed)
            logger.debug("processing: %s" % url)
            #self.processed.append(url)
            if isinstance(url, str):
                logger.debug("url is a string")
                self.url = url
                (data, content_type) = self.retrieve_data()
            else:
                self.url = None
                data = ""
            if self.crawl:
                self.get_urls(data)
            self.process_data(data, content_type)
            url = self.get_next()

    def retrieve_data(self):
        """
        Retrieve the data from the website.
        """
        w = WebGrabber(self.url)
        w.get_page()
        if w.done():
            return (w.get_data(), w.get_content_type())
        else:
            logger.warning("Using empty string as WebSite data")
            return ("", None)

    def process_data(self, html, content_type):
        """
        Do the web grab from the website and all the filtering
        and searching.
        """
        #### TO DO add processing for other content types
        self.process_as_text(html)

    def process_as_text(self, html):
        """
        Process the text data received.
        """
        logger.debug("processing:\n%s..." % (html[:25] or "Nothing")) 
        (search, scrub, reverse) = self.generate_filters()
        logger.debug("Search: %s" % search)
        logger.debug("Scrub: %s" % scrub)
        logger.debug("reverse: %s" % reverse)
        if reverse == 0:
            if scrub is not None:
                logger.debug("Scrubbing data")
                html = scrub.scrub(html)
                logger.debug("New size: %d" % len(html))
            if search is not None:
                logger.debug("Searching data")
                logger.debug("Working this %s" % type(html))
                html = search.keep(html)
                logger.debug(html)
        else:
            if search is not None:
                logger.debug("Searching data")
                html = search.keep(html)
            if scrub is not None:
                logger.debug("Scrubbing data")
                html = scrub.scrub(html)
        if not self.crawl or self.webData is None:
            self.webData = html
        elif isinstance(self.webData, str):
            self.webData += "\n\n####\n\n" + html
        elif isinstance(self.webData, set):
            self.webData.append(html)
        #print "-------"
        #print self.webData[:100]
    
    def generate_filters(self):
        """
        Generate the list of filters from the configutation.
        """
        if self.config == None:
            return (None, None, 0)
        if self.config.has_search():
            search = KeeperReg(self.config.search_string)
        else:
            search = None
        if self.config.has_filter():
            scrub = ScrubReg(self.config.filter_string)
        else:
            scrub = None
        return (search, scrub, self.config.get_reverse())

    def get_data(self):
        """
        @return: The data from the web page
        """
        #if len(self.webData) == 1:
        #    return self.webData[0]
        if self.webData is None:
            return ""
        return self.webData

    def output_file(self, name):
        """
        generate the output file with the user specified name.

        @param name: The file name to be used
        """
        if os.path.exists(name):
            count = 1
            tmp = "%s_%d" % (name, count)
            while os.path.exists(tmp):
                count += 1
                tmp = "%s_%d" % (name,count)
            logger.warning(("unable to use '%s' as an output " +
                            "using '%s' instead.") % (name, tmp)) 
            name = tmp
        output = open(name, "w")
        #print self.get_data()
        if isinstance(self.get_data(), list):
            for l in self.get_data():
                output.write("%s\n" % l)
        else:
            output.write(str(self.get_data()))
        output.close()