def _download_from_dirport(url, compression, timeout):
"""
Downloads descriptors from the given url.
:param str url: dirport url from which to download from
:param list compression: compression methods for the request
:param float timeout: duration before we'll time out our request
:returns: two value tuple of the form (data, reply_headers)
:raises:
* :class:`~stem.DownloadTimeout` if our request timed out
* :class:`~stem.DownloadFailed` if our request fails
"""
try:
response = urllib.urlopen(
urllib.Request(
url,
headers = {
'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)),
'User-Agent': stem.USER_AGENT,
}
),
timeout = timeout,
)
except socket.timeout as exc:
raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout)
except:
exc, stacktrace = sys.exc_info()[1:3]
raise stem.DownloadFailed(url, exc, stacktrace)
return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers
def download_man_page(path: Optional[str] = None, file_handle: Optional[BinaryIO] = None, url: str = GITWEB_MANUAL_URL, timeout: int = 20) -> None:
"""
Downloads tor's latest man page from `gitweb.torproject.org
<https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. This method is
both slow and unreliable - please see the warnings on
:func:`~stem.manual.Manual.from_remote`.
:param path: path to save tor's man page to
:param file_handle: file handler to save tor's man page to
:param url: url to download tor's asciidoc manual from
:param timeout: seconds to wait before timing out the request
:raises: **IOError** if unable to retrieve the manual
"""
if not path and not file_handle:
raise ValueError("Either the path or file_handle we're saving to must be provided")
elif not stem.util.system.is_available('a2x'):
raise IOError('We require a2x from asciidoc to provide a man page')
with tempfile.TemporaryDirectory() as dirpath:
asciidoc_path = os.path.join(dirpath, 'tor.1.txt')
manual_path = os.path.join(dirpath, 'tor.1')
try:
with open(asciidoc_path, 'wb') as asciidoc_file:
request = urllib.request.urlopen(url, timeout = timeout)
shutil.copyfileobj(request, asciidoc_file)
except:
exc, stacktrace = sys.exc_info()[1:3]
message = "Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc)
raise stem.DownloadFailed(url, exc, stacktrace, message)
try:
stem.util.system.call('a2x -f manpage %s' % asciidoc_path)
if not os.path.exists(manual_path):
raise OSError('no man page was generated')
except stem.util.system.CallError as exc:
raise IOError("Unable to run '%s': %s" % (exc.command, stem.util.str_tools._to_unicode(exc.stderr)))
if path:
try:
path_dir = os.path.dirname(path)
if not os.path.exists(path_dir):
os.makedirs(path_dir)
shutil.copyfile(manual_path, path)
except OSError as exc:
raise IOError(exc)
if file_handle:
with open(manual_path, 'rb') as manual_file:
shutil.copyfileobj(manual_file, file_handle)
file_handle.flush()
def from_remote(timeout: int = 60) -> Dict[str, 'stem.directory.Fallback']:
try:
lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines()
if not lines:
raise IOError('no content')
except:
exc, stacktrace = sys.exc_info()[1:3]
message = "Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_URL, exc)
raise stem.DownloadFailed(GITWEB_FALLBACK_URL, exc, stacktrace, message)
# header metadata
if lines[0] != '/* type=fallback */':
raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_URL)
header = {}
for line in Fallback._pop_section(lines):
mapping = FALLBACK_MAPPING.match(line)
if mapping:
header[mapping.group(1)] = mapping.group(2)
else:
raise IOError('Malformed fallback directory header line: %s' % line)
Fallback._pop_section(lines) # skip human readable comments
# Entries look like...
#
# "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB"
# " ipv6=[2a01:4f8:162:51e2::2]:9001"
# /* nickname=rueckgrat */
# /* extrainfo=1 */
try:
results = {}
for matches in _directory_entries(lines, Fallback._pop_section, (FALLBACK_ADDR, FALLBACK_NICKNAME, FALLBACK_EXTRAINFO, FALLBACK_IPV6), required = (FALLBACK_ADDR,)):
address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR]
results[fingerprint] = Fallback(
address = address,
or_port = int(or_port),
dir_port = int(dir_port),
fingerprint = fingerprint,
nickname = matches.get(FALLBACK_NICKNAME), # type: ignore
has_extrainfo = matches.get(FALLBACK_EXTRAINFO) == '1',
orport_v6 = matches.get(FALLBACK_IPV6), # type: ignore
header = header,
)
except ValueError as exc:
raise IOError(str(exc))
return results
def from_remote(
timeout: int = 60) -> Dict[str, 'stem.directory.Authority']:
try:
lines = str_tools._to_unicode(
urllib.request.urlopen(GITWEB_AUTHORITY_URL,
timeout=timeout).read()).splitlines()
if not lines:
raise OSError('no content')
except:
exc, stacktrace = sys.exc_info()[1:3]
message = "Unable to download tor's directory authorities from %s: %s" % (
GITWEB_AUTHORITY_URL, exc)
raise stem.DownloadFailed(GITWEB_AUTHORITY_URL, exc, stacktrace,
message)
# Entries look like...
#
# "moria1 orport=9101 "
# "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 "
# "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31",
try:
results = {}
for matches in _directory_entries(
lines,
Authority._pop_section, (AUTHORITY_NAME, AUTHORITY_V3IDENT,
AUTHORITY_IPV6, AUTHORITY_ADDR),
required=(AUTHORITY_NAME, AUTHORITY_ADDR)):
nickname, or_port = matches.get(AUTHORITY_NAME) # type: ignore
address, dir_port, fingerprint = matches.get(
AUTHORITY_ADDR) # type: ignore
results[nickname] = Authority(
address=address,
or_port=or_port,
dir_port=dir_port,
fingerprint=fingerprint.replace(' ', ''),
nickname=nickname,
orport_v6=matches.get(AUTHORITY_IPV6), # type: ignore
v3ident=matches.get(AUTHORITY_V3IDENT), # type: ignore
)
except ValueError as exc:
raise OSError(str(exc))
return results
def download(url: str,
timeout: Optional[float] = None,
retries: Optional[int] = None) -> bytes:
"""
Download from the given url.
.. versionadded:: 1.8.0
:param url: uncompressed url to download from
:param timeout: timeout when connection becomes idle, no timeout
applied if **None**
:param retires: maximum attempts to impose
:returns: **bytes** content of the given url
:raises:
* :class:`~stem.DownloadTimeout` if our request timed out
* :class:`~stem.DownloadFailed` if our request fails
"""
if retries is None:
retries = 0
start_time = time.time()
try:
return urllib.request.urlopen(url, timeout=timeout).read()
except socket.timeout as exc:
raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout)
except:
exception, stacktrace = sys.exc_info()[1:3]
if timeout is not None:
timeout -= time.time() - start_time
if retries > 0 and (timeout is None or timeout > 0):
log.debug('Failed to download from %s (%i retries remaining): %s' %
(url, retries, exception))
return download(url, timeout, retries - 1)
else:
log.debug('Failed to download from %s: %s' % (url, exception))
raise stem.DownloadFailed(url, exception, stacktrace)
