def _download_from_dirport(url, compression, timeout): """ Downloads descriptors from the given url. :param str url: dirport url from which to download from :param list compression: compression methods for the request :param float timeout: duration before we'll time out our request :returns: two value tuple of the form (data, reply_headers) :raises: * :class:`~stem.DownloadTimeout` if our request timed out * :class:`~stem.DownloadFailed` if our request fails """ try: response = urllib.urlopen( urllib.Request( url, headers = { 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), 'User-Agent': stem.USER_AGENT, } ), timeout = timeout, ) except socket.timeout as exc: raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) except: exc, stacktrace = sys.exc_info()[1:3] raise stem.DownloadFailed(url, exc, stacktrace) return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers
def download_man_page(path: Optional[str] = None, file_handle: Optional[BinaryIO] = None, url: str = GITWEB_MANUAL_URL, timeout: int = 20) -> None: """ Downloads tor's latest man page from `gitweb.torproject.org <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. This method is both slow and unreliable - please see the warnings on :func:`~stem.manual.Manual.from_remote`. :param path: path to save tor's man page to :param file_handle: file handler to save tor's man page to :param url: url to download tor's asciidoc manual from :param timeout: seconds to wait before timing out the request :raises: **IOError** if unable to retrieve the manual """ if not path and not file_handle: raise ValueError("Either the path or file_handle we're saving to must be provided") elif not stem.util.system.is_available('a2x'): raise IOError('We require a2x from asciidoc to provide a man page') with tempfile.TemporaryDirectory() as dirpath: asciidoc_path = os.path.join(dirpath, 'tor.1.txt') manual_path = os.path.join(dirpath, 'tor.1') try: with open(asciidoc_path, 'wb') as asciidoc_file: request = urllib.request.urlopen(url, timeout = timeout) shutil.copyfileobj(request, asciidoc_file) except: exc, stacktrace = sys.exc_info()[1:3] message = "Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc) raise stem.DownloadFailed(url, exc, stacktrace, message) try: stem.util.system.call('a2x -f manpage %s' % asciidoc_path) if not os.path.exists(manual_path): raise OSError('no man page was generated') except stem.util.system.CallError as exc: raise IOError("Unable to run '%s': %s" % (exc.command, stem.util.str_tools._to_unicode(exc.stderr))) if path: try: path_dir = os.path.dirname(path) if not os.path.exists(path_dir): os.makedirs(path_dir) shutil.copyfile(manual_path, path) except OSError as exc: raise IOError(exc) if file_handle: with open(manual_path, 'rb') as manual_file: shutil.copyfileobj(manual_file, file_handle) file_handle.flush()
def from_remote(timeout: int = 60) -> Dict[str, 'stem.directory.Fallback']: try: lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() if not lines: raise IOError('no content') except: exc, stacktrace = sys.exc_info()[1:3] message = "Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc) raise stem.DownloadFailed(GITWEB_FALLBACK_URL, exc, stacktrace, message) # header metadata if lines[0] != '/* type=fallback */': raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL) header = {} for line in Fallback._pop_section(lines): mapping = FALLBACK_MAPPING.match(line) if mapping: header[mapping.group(1)] = mapping.group(2) else: raise IOError('Malformed fallback directory header line: %s' % line) Fallback._pop_section(lines) # skip human readable comments # Entries look like... # # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" # " ipv6=[2a01:4f8:162:51e2::2]:9001" # /* nickname=rueckgrat */ # /* extrainfo=1 */ try: results = {} for matches in _directory_entries(lines, Fallback._pop_section, (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6), required = (FALLBACK_ADDR,)): address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] results[fingerprint] = Fallback( address = address, or_port = int(or_port), dir_port = int(dir_port), fingerprint = fingerprint, nickname = matches.get(FALLBACK_NICKNAME), # type: ignore has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1', orport_v6 = matches.get(FALLBACK_IPV6), # type: ignore header = header, ) except ValueError as exc: raise IOError(str(exc)) return results
def from_remote( timeout: int = 60) -> Dict[str, 'stem.directory.Authority']: try: lines = str_tools._to_unicode( urllib.request.urlopen(GITWEB_AUTHORITY_URL, timeout=timeout).read()).splitlines() if not lines: raise OSError('no content') except: exc, stacktrace = sys.exc_info()[1:3] message = "Unable to download tor's directory authorities from %s: %s" % ( GITWEB_AUTHORITY_URL, exc) raise stem.DownloadFailed(GITWEB_AUTHORITY_URL, exc, stacktrace, message) # Entries look like... # # "moria1 orport=9101 " # "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " # "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", try: results = {} for matches in _directory_entries( lines, Authority._pop_section, (AUTHORITY_NAME, AUTHORITY_V3IDENT, AUTHORITY_IPV6, AUTHORITY_ADDR), required=(AUTHORITY_NAME, AUTHORITY_ADDR)): nickname, or_port = matches.get(AUTHORITY_NAME) # type: ignore address, dir_port, fingerprint = matches.get( AUTHORITY_ADDR) # type: ignore results[nickname] = Authority( address=address, or_port=or_port, dir_port=dir_port, fingerprint=fingerprint.replace(' ', ''), nickname=nickname, orport_v6=matches.get(AUTHORITY_IPV6), # type: ignore v3ident=matches.get(AUTHORITY_V3IDENT), # type: ignore ) except ValueError as exc: raise OSError(str(exc)) return results
def download(url: str, timeout: Optional[float] = None, retries: Optional[int] = None) -> bytes: """ Download from the given url. .. versionadded:: 1.8.0 :param url: uncompressed url to download from :param timeout: timeout when connection becomes idle, no timeout applied if **None** :param retires: maximum attempts to impose :returns: **bytes** content of the given url :raises: * :class:`~stem.DownloadTimeout` if our request timed out * :class:`~stem.DownloadFailed` if our request fails """ if retries is None: retries = 0 start_time = time.time() try: return urllib.request.urlopen(url, timeout=timeout).read() except socket.timeout as exc: raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) except: exception, stacktrace = sys.exc_info()[1:3] if timeout is not None: timeout -= time.time() - start_time if retries > 0 and (timeout is None or timeout > 0): log.debug('Failed to download from %s (%i retries remaining): %s' % (url, retries, exception)) return download(url, timeout, retries - 1) else: log.debug('Failed to download from %s: %s' % (url, exception)) raise stem.DownloadFailed(url, exception, stacktrace)