def _determine_redirect(self, url, verb, timeout=15, headers={}): """ Internal redirect function, focuses on HTTP and worries less about application-y stuff. @param url: the url to check @param verb: the verb, e.g. head, or get. @param timeout: the time, in seconds, that requests should wait before throwing an exception. @param headers: a set of headers as expected by requests. @return: the url that needs to be scanned. It may be equal to the url parameter if no redirect is needed. """ requests_verb = getattr(self.session, verb) r = requests_verb(url, timeout=timeout, headers=headers, allow_redirects=False) redirect = 300 <= r.status_code < 400 url_new = url if redirect: redirect_url = r.headers['Location'] url_new = redirect_url relative_redirect = not redirect_url.startswith('http') if relative_redirect: url_new = url base_redir = base_url(redirect_url) base_supplied = base_url(url) same_base = base_redir == base_supplied if same_base: url_new = url return url_new