def crawlingScan(self, url, apiCalls = [], allFoundURLs = []):
self.count = self.count - 1
if self.count < 0:
return
harParser = HarParser(self.harDirectory, searchString=self.searchString, removeParams=self.removeParams)
#If uncommented, will return as soon as a matching call is found
#if self.searchString is not None and len(apiCalls) > 0:
# return apiCalls
try:
print("Scanning URL: "+url)
html = self.openURL(url)
if html is not None:
bsObj = BeautifulSoup(html, "lxml")
harObj = harParser.getSingleHarFile()
apiCalls = harParser.scanHarfile(harObj, apiCalls=apiCalls)
allFoundURLs, newUrls = self.findInternalURLs(bsObj, url, allFoundURLs)
shuffle(newUrls)
for newUrl in newUrls:
self.crawlingScan(newUrl, apiCalls, allFoundURLs)
except (KeyboardInterrupt, SystemExit):
print("Stopping crawl")
self.browser.close()
apiWriter = APIWriter(apiCalls)
apiWriter.outputAPIs()
exit(1)
return apiCalls
def crawlingScan(self, url, apiCalls=[], allFoundURLs=[]):
self.count = self.count - 1
if self.count < 0:
return
harParser = HarParser(self.harDirectory,
searchString=self.searchString,
removeParams=self.removeParams)
#If uncommented, will return as soon as a matching call is found
#if self.searchString is not None and len(apiCalls) > 0:
# return apiCalls
try:
print("Scanning URL: " + url)
html = self.openURL(url)
if html is not None:
bsObj = BeautifulSoup(html, "lxml")
harObj = harParser.getSingleHarFile()
apiCalls = harParser.scanHarfile(harObj, apiCalls=apiCalls)
allFoundURLs, newUrls = self.findInternalURLs(
bsObj, url, allFoundURLs)
shuffle(newUrls)
for newUrl in newUrls:
self.crawlingScan(newUrl, apiCalls, allFoundURLs)
except (KeyboardInterrupt, SystemExit):
print("Stopping crawl")
self.browser.close()
apiWriter = APIWriter(apiCalls)
apiWriter.outputAPIs()
exit(1)
return apiCalls
"File containing JSON formatted cookies to set in driver (with target URL only)",
nargs='?')
parser.add_argument("-i",
help="Count of pages to crawl (with target URL only)",
nargs='?')
parser.add_argument(
'--p',
help=
"Flag, remove unnecessary parameters (may dramatically increase run time)",
action='store_true')
args = parser.parse_args()
if not (args.u or args.d):
print("Need to provide either a URL or directory or both. Use -h for help")
sys.exit(1)
#Default to directory name "hars" and count of 1
directory = "hars" if args.d is None else args.d
count = 1 if args.i is None else int(args.i)
finder = APIFinder(url=args.u,
harDirectory=directory,
searchString=args.s,
removeParams=args.p,
count=count,
cookies=args.c)
apiCalls = finder.start()
apiWriter = APIWriter(apiCalls)
apiWriter.outputAPIs()
from apicall import APIWriter
from apiFinder import APIFinder
import sys
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-u", help="Target URL. If not provided, target directory will be scanned for har files.", nargs='?')
parser.add_argument("-d", help="Target directory (default is \"hars\"). If URL is provided, directory will store har files. If URL is not provided, directory will be scanned. ", nargs='?')
parser.add_argument("-s", help="Search term", nargs='?')
parser.add_argument("-c", help="File containing JSON formatted cookies to set in driver (with target URL only)", nargs='?')
parser.add_argument("-i", help="Count of pages to crawl (with target URL only)", nargs='?')
parser.add_argument('--p', help="Flag, remove unnecessary parameters (may dramatically increase run time)", action='store_true')
args = parser.parse_args()
if not (args.u or args.d):
print("Need to provide either a URL or directory or both. Use -h for help")
sys.exit(1)
#Default to directory name "hars" and count of 1
directory = "hars" if args.d is None else args.d
count = 1 if args.i is None else int(args.i)
finder = APIFinder(url=args.u, harDirectory=directory, searchString=args.s, removeParams=args.p, count=count, cookies=args.c)
apiCalls = finder.start()
apiWriter = APIWriter(apiCalls)
apiWriter.outputAPIs()
