def create_scrapelist(): scrape_links = [] global scrape_list # go back to case crawled directory to store scrape results in crawl folder dir_crawled(base, case) # list the directories for each project folder for root, dirs, files in os.walk(os.getcwd(), topdown=False): for name in dirs: directory_list.append(os.path.join(root, name)) # collect urls from all crawled.txt files and append to scrapelist for root, name in enumerate(directory_list): if (case + '_crawler') in name: file = Path.cwd().joinpath(name).joinpath("crawled.txt") with open(file) as f: for url in f: scrape_links.append(url) # clear directory list, remove duplicates and go back to case folder directory_list.clear() scrape_list = list(dict.fromkeys(scrape_links)) dir_case(base, case) # create scraper folder to save scrapelist to, then go back to temp dir create_project_dir(folder) dir_scrapelist(base, case) pickle_file("scrape_list.txt", scrape_list) dir_case(base, case)
def __init__(self): with open('./config/config.json', 'r') as myfile: data = myfile.read() sqlmap.config = loads(data) create_project_dir(sqlmap.config["ProjectName"] + "/sqlmap") sqlmap.files = create_penetration_data_files( sqlmap.config["ProjectName"], "sqlmap") sqlmap.queue = file_to_set(sqlmap.files[0]) sqlmap.neg = file_to_set(sqlmap.files[1]) sqlmap.pos = file_to_set(sqlmap.files[2]) sqlmap.startTime = time() sqlmap.work()
def __init__(self): with open('./config/config.json', 'r') as myfile: data = myfile.read() xss.config = loads(data) create_project_dir(xss.config["ProjectName"] + "/xss") xss.files = create_penetration_data_files(xss.config["ProjectName"], "xss") xss.queue = file_to_set(xss.files[0]) xss.neg = file_to_set(xss.files[1]) xss.pos = file_to_set(xss.files[2]) xss.startTime = time() xss.work()
def boot(): create_project_dir(Spider.project_name) create_data_files(Spider.project_name, Spider.base_url) Spider.queue = file_to_set(Spider.queue_file) Spider.crawled = file_to_set(Spider.crawled_file) Spider.finish = file_to_set(Spider.finish_file)