def run_module(self, url, home, depth_limit):

        confine_prefix = url
        exclude = ''

        if home[-1] != '/': home = home + '/'

        self.mprint('[%s] Crawling paths in %s (depth %i)' % (self.name, url, depth_limit))

        try:
            crawler = Crawler(url, depth_limit, confine_prefix, exclude)
            crawler.crawl()
        except Exception, e:
            raise ModuleException(self.name, "Crawler exception: %s" % str(e))
Beispiel #2
0
    def run_module(self, url, home, depth_limit):
        
        confine_prefix = url
        exclude = ''

        if home[-1] != '/': home = home + '/'

        self.mprint('[%s] Crawling paths in %s (depth %i)' % (self.name, url, depth_limit))
        
        try:
            crawler = Crawler(url, depth_limit, confine_prefix, exclude)
            crawler.crawl()
        except Exception, e:
            raise ModuleException(self.name, "Crawler exception: %s" % str(e))
Beispiel #3
0
 def _prepare(self):
 
     if not url_validator.match(self.args['url']):
         raise ProbeException(self.name, '\'%s\': %s' % (self.args['url'], WARN_NOT_URL) )
     if not url_validator.match(self.args['baseurl']):
         raise ProbeException(self.name, '\'%s\': %s' % (self.args['baseurl'], WARN_NOT_URL) )
 
     url = self.args['url']    
     baseurl = self.args['baseurl']
     rpath = self.args['rpath']
     
     urls = []
 
     try:
         crawler = Crawler(url, self.args['depth'], '', '')
         crawler.crawl()
     except Exception, e:
         raise ProbeException(self.name, "%s: %s" % (ERR_CRAWLER_EXCEPT, str(e)))
Beispiel #4
0
    def _prepare(self):

        if not url_validator.match(self.args["url"]):
            raise ProbeException(self.name, "'%s': %s" % (self.args["url"], WARN_NOT_URL))
        if not url_validator.match(self.args["baseurl"]):
            raise ProbeException(self.name, "'%s': %s" % (self.args["baseurl"], WARN_NOT_URL))

        url = self.args["url"]
        baseurl = self.args["baseurl"]
        rpath = self.args["rpath"]

        urls = []

        try:
            crawler = Crawler(url, self.args["depth"], "", "")
            crawler.crawl()
        except ModuleException, e:
            raise
Beispiel #5
0
 def _prepare(self):
 
     if not url_validator.match(self.args['url']):
         raise ProbeException(self.name, '\'%s\': %s' % (self.args['url'], WARN_NOT_URL) )
     if not url_validator.match(self.args['baseurl']):
         raise ProbeException(self.name, '\'%s\': %s' % (self.args['baseurl'], WARN_NOT_URL) )
 
     url = self.args['url']    
     baseurl = self.args['baseurl']
     rpath = self.args['rpath']
     
     urls = []
 
     try:
         crawler = Crawler(url, self.args['depth'], '', '')
         crawler.crawl()
     except ModuleException, e:
         raise