def crawlingScan(self, url, apiCalls = [], allFoundURLs = []): self.count = self.count - 1 if self.count < 0: return harParser = HarParser(self.harDirectory, searchString=self.searchString, removeParams=self.removeParams) #If uncommented, will return as soon as a matching call is found #if self.searchString is not None and len(apiCalls) > 0: # return apiCalls try: print("Scanning URL: "+url) html = self.openURL(url) if html is not None: bsObj = BeautifulSoup(html, "lxml") harObj = harParser.getSingleHarFile() apiCalls = harParser.scanHarfile(harObj, apiCalls=apiCalls) allFoundURLs, newUrls = self.findInternalURLs(bsObj, url, allFoundURLs) shuffle(newUrls) for newUrl in newUrls: self.crawlingScan(newUrl, apiCalls, allFoundURLs) except (KeyboardInterrupt, SystemExit): print("Stopping crawl") self.browser.close() apiWriter = APIWriter(apiCalls) apiWriter.outputAPIs() exit(1) return apiCalls
def crawlingScan(self, url, apiCalls=[], allFoundURLs=[]): self.count = self.count - 1 if self.count < 0: return harParser = HarParser(self.harDirectory, searchString=self.searchString, removeParams=self.removeParams) #If uncommented, will return as soon as a matching call is found #if self.searchString is not None and len(apiCalls) > 0: # return apiCalls try: print("Scanning URL: " + url) html = self.openURL(url) if html is not None: bsObj = BeautifulSoup(html, "lxml") harObj = harParser.getSingleHarFile() apiCalls = harParser.scanHarfile(harObj, apiCalls=apiCalls) allFoundURLs, newUrls = self.findInternalURLs( bsObj, url, allFoundURLs) shuffle(newUrls) for newUrl in newUrls: self.crawlingScan(newUrl, apiCalls, allFoundURLs) except (KeyboardInterrupt, SystemExit): print("Stopping crawl") self.browser.close() apiWriter = APIWriter(apiCalls) apiWriter.outputAPIs() exit(1) return apiCalls