def test_jquery_re(self): urls = set() merged_strings = self.get_file_contents('jquery.js') for x in URL_RE.findall(merged_strings): try: urls.add(URL(x[0])) except ValueError: pass return urls
def extract_full_urls(self): urls = set() merged_strings = ' \n'.join(self.get_strings()) for x in URL_RE.findall(merged_strings): try: urls.add(URL(x[0])) except ValueError: pass return urls
def _extract_full_urls(self, doc_string): """ Detect full URLs, which look like http://foo/bar?id=1 """ for url_mo in URL_RE.finditer(doc_string): if self._require_quotes: if not self._is_quoted(url_mo, doc_string): continue try: url = URL(url_mo.group(0), encoding=self._encoding) except ValueError: pass else: self._re_urls.add(url)