def create_scrapelist():
scrape_links = []
global scrape_list
# go back to case crawled directory to store scrape results in crawl folder
dir_crawled(base, case)
# list the directories for each project folder
for root, dirs, files in os.walk(os.getcwd(), topdown=False):
for name in dirs:
directory_list.append(os.path.join(root, name))
# collect urls from all crawled.txt files and append to scrapelist
for root, name in enumerate(directory_list):
if (case + '_crawler') in name:
file = Path.cwd().joinpath(name).joinpath("crawled.txt")
with open(file) as f:
for url in f:
scrape_links.append(url)
# clear directory list, remove duplicates and go back to case folder
directory_list.clear()
scrape_list = list(dict.fromkeys(scrape_links))
dir_case(base, case)
# create scraper folder to save scrapelist to, then go back to temp dir
create_project_dir(folder)
dir_scrapelist(base, case)
pickle_file("scrape_list.txt", scrape_list)
dir_case(base, case)
def __init__(self):
with open('./config/config.json', 'r') as myfile:
data = myfile.read()
sqlmap.config = loads(data)
create_project_dir(sqlmap.config["ProjectName"] + "/sqlmap")
sqlmap.files = create_penetration_data_files(
sqlmap.config["ProjectName"], "sqlmap")
sqlmap.queue = file_to_set(sqlmap.files[0])
sqlmap.neg = file_to_set(sqlmap.files[1])
sqlmap.pos = file_to_set(sqlmap.files[2])
sqlmap.startTime = time()
sqlmap.work()
def __init__(self):
with open('./config/config.json', 'r') as myfile:
data = myfile.read()
xss.config = loads(data)
create_project_dir(xss.config["ProjectName"] + "/xss")
xss.files = create_penetration_data_files(xss.config["ProjectName"],
"xss")
xss.queue = file_to_set(xss.files[0])
xss.neg = file_to_set(xss.files[1])
xss.pos = file_to_set(xss.files[2])
xss.startTime = time()
xss.work()
def boot():
create_project_dir(Spider.project_name)
create_data_files(Spider.project_name, Spider.base_url)
Spider.queue = file_to_set(Spider.queue_file)
Spider.crawled = file_to_set(Spider.crawled_file)
Spider.finish = file_to_set(Spider.finish_file)