def __init__(self, config: Config) -> None:
"""
Initialize Tor Launcher
:param config: The config class instance that contains global configuration values
:type config: Config
:raises TorLauncherInitError: If Tor launcher wasn't able connect to the Tor container
"""
# Public class attributes
self.ip_address: str
self.socks_port: int
self.control_port: int
self.relay_fingerprints: List[Any]
# Private class attributes
self.__logger = logging.getLogger(__name__)
self.__config: Config = config
self.__circuit_id: Controller.new_circuit
self.__num_retries_on_fail: int = 3
self.__delay_in_seconds_between_retries: int = 3
try:
self.__docker_network_name = self.__config["docker_network"]
except Exception as exception:
self.__logger.warning("Could not connect to Tor:\n %s", exception)
raise TorLauncherInitError from exception
# Silence the stem logger
stem_logger = get_logger()
stem_logger.propagate = False
# Execute the private methods
self.__launch_tor_container()
self.__bind_stem_to_tor_container()
def __init__(self):
self.hostname = Config["proxy"]["hostname"]
self.socks_port = Config["proxy"]["socks_port"]
self.ctrl_port = Config["proxy"]["control_port"]
self.password = Config["proxy"]["password"]
self.controller = None
self.ip = ""
logger = get_logger()
logger.level = logging.WARNING
def test_is_tracing(self):
logger = log.get_logger()
original_handlers = logger.handlers
logger.handlers = [log._NullHandler()]
try:
self.assertFalse(log.is_tracing())
logger.addHandler(log.LogBuffer(log.DEBUG))
self.assertFalse(log.is_tracing())
logger.addHandler(log.LogBuffer(log.TRACE))
self.assertTrue(log.is_tracing())
finally:
logger.handlers = original_handlers
def test_all_use_cases(self, authenticate_safecookie_mock,
authenticate_cookie_mock,
authenticate_password_mock, authenticate_none_mock):
"""
Does basic validation that all valid use cases for the PROTOCOLINFO input
and dependent functions result in either success or a AuthenticationFailed
subclass being raised.
"""
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# exceptions that the authentication functions are documented to raise
all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None))
all_auth_password_exc = (None,
stem.connection.PasswordAuthRejected(None),
stem.connection.IncorrectPassword(None))
all_auth_cookie_exc = (
None, stem.connection.IncorrectCookieSize(None, False, None),
stem.connection.UnreadableCookieFile(None, False, None),
stem.connection.CookieAuthRejected(None, False, None),
stem.connection.IncorrectCookieValue(None, False, None),
stem.connection.UnrecognizedAuthChallengeMethod(None, None, None),
stem.connection.AuthChallengeFailed(None, None),
stem.connection.AuthSecurityFailure(None, None),
stem.connection.InvalidClientNonce(None, None))
# authentication functions might raise a controller error when
# 'suppress_ctl_errors' is False, so including those
control_exc = (stem.ProtocolError(None), stem.SocketError(None),
stem.SocketClosed(None))
all_auth_none_exc += control_exc
all_auth_password_exc += control_exc
all_auth_cookie_exc += control_exc
auth_method_combinations = mocking.get_all_combinations(
[
stem.connection.AuthMethod.NONE,
stem.connection.AuthMethod.PASSWORD,
stem.connection.AuthMethod.COOKIE,
stem.connection.AuthMethod.SAFECOOKIE,
stem.connection.AuthMethod.UNKNOWN,
],
include_empty=True)
for protocolinfo_auth_methods in auth_method_combinations:
# protocolinfo input for the authenticate() call we'll be making
protocolinfo_arg = mocking.get_protocolinfo_response(
auth_methods=protocolinfo_auth_methods,
cookie_path='/tmp/blah',
)
for auth_none_exc in all_auth_none_exc:
for auth_password_exc in all_auth_password_exc:
for auth_cookie_exc in all_auth_cookie_exc:
# Determine if the authenticate() call will succeed and mock each
# of the authenticate_* function to raise its given exception.
#
# This implementation is slightly inaccurate in a couple regards...
# a. it raises safecookie exceptions from authenticate_cookie()
# b. exceptions raised by authenticate_cookie() and
# authenticate_safecookie() are always the same
#
# However, adding another loop for safe_cookie exceptions means
# multiplying our runtime many fold. This exercises everything that
# matters so the above inaccuracies seem fine.
expect_success = False
auth_mocks = {
stem.connection.AuthMethod.NONE:
(authenticate_none_mock, auth_none_exc),
stem.connection.AuthMethod.PASSWORD:
(authenticate_password_mock, auth_password_exc),
stem.connection.AuthMethod.COOKIE:
(authenticate_cookie_mock, auth_cookie_exc),
stem.connection.AuthMethod.SAFECOOKIE:
(authenticate_safecookie_mock, auth_cookie_exc),
}
for auth_method in auth_mocks:
auth_mock, raised_exc = auth_mocks[auth_method]
if not raised_exc:
# Mocking this authentication method so it will succeed. If
# it's among the protocolinfo methods then expect success.
auth_mock.side_effect = None
expect_success |= auth_method in protocolinfo_auth_methods
else:
auth_mock.side_effect = raised_exc
if expect_success:
stem.connection.authenticate(
None, 'blah', None, protocolinfo_arg)
else:
self.assertRaises(
stem.connection.AuthenticationFailure,
stem.connection.authenticate, None, 'blah',
None, protocolinfo_arg)
# revert logging back to normal
stem_logger.setLevel(log.logging_level(log.TRACE))
def test_all_use_cases(self):
"""
Does basic validation that all valid use cases for the PROTOCOLINFO input
and dependent functions result in either success or a AuthenticationFailed
subclass being raised.
"""
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# exceptions that the authentication functions are documented to raise
all_auth_none_exc = (None,
stem.connection.OpenAuthRejected(None))
all_auth_password_exc = (None,
stem.connection.PasswordAuthRejected(None),
stem.connection.IncorrectPassword(None))
all_auth_cookie_exc = (None,
stem.connection.IncorrectCookieSize(None, None),
stem.connection.UnreadableCookieFile(None, None),
stem.connection.CookieAuthRejected(None, None),
stem.connection.IncorrectCookieValue(None, None))
# authentication functions might raise a controller error when
# 'suppress_ctl_errors' is False, so including those
control_exc = (
stem.socket.ProtocolError(None),
stem.socket.SocketError(None),
stem.socket.SocketClosed(None))
all_auth_none_exc += control_exc
all_auth_password_exc += control_exc
all_auth_cookie_exc += control_exc
for protocolinfo_auth_methods in _get_all_auth_method_combinations():
# protocolinfo input for the authenticate() call we'll be making
protocolinfo_arg = mocking.get_protocolinfo_response(
auth_methods = protocolinfo_auth_methods,
cookie_path = "/tmp/blah",
)
for auth_none_exc in all_auth_none_exc:
for auth_password_exc in all_auth_password_exc:
for auth_cookie_exc in all_auth_cookie_exc:
# determine if the authenticate() call will succeed and mock each
# of the authenticate_* function to raise its given exception
expect_success = False
auth_mocks = {
stem.connection.AuthMethod.NONE:
(stem.connection.authenticate_none, auth_none_exc),
stem.connection.AuthMethod.PASSWORD:
(stem.connection.authenticate_password, auth_password_exc),
stem.connection.AuthMethod.COOKIE:
(stem.connection.authenticate_cookie, auth_cookie_exc),
}
for auth_method in auth_mocks:
auth_function, raised_exc = auth_mocks[auth_method]
if not raised_exc:
# Mocking this authentication method so it will succeed. If
# it's among the protocolinfo methods then expect success.
mocking.mock(auth_function, mocking.no_op())
expect_success |= auth_method in protocolinfo_auth_methods
else:
mocking.mock(auth_function, mocking.raise_exception(raised_exc))
if expect_success:
stem.connection.authenticate(None, "blah", None, protocolinfo_arg)
else:
self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, "blah", None, protocolinfo_arg)
# revert logging back to normal
stem_logger.setLevel(log.logging_level(log.TRACE))
def test_all_use_cases(self, authenticate_safecookie_mock,
authenticate_cookie_mock,
authenticate_password_mock, authenticate_none_mock):
"""
Does basic validation that all valid use cases for the PROTOCOLINFO input
and dependent functions result in either success or a AuthenticationFailed
subclass being raised.
"""
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# exceptions that the authentication functions are documented to raise
all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None),
stem.ControllerError(None))
all_auth_password_exc = (None,
stem.connection.PasswordAuthRejected(None),
stem.connection.IncorrectPassword(None),
stem.ControllerError(None))
all_auth_cookie_exc = (None,
stem.connection.CookieAuthRejected(
None, False, None),
stem.connection.IncorrectCookieValue(
None, False, None),
stem.connection.IncorrectCookieSize(
None, False, None),
stem.connection.UnreadableCookieFile(
None, False, None),
stem.connection.AuthChallengeFailed(None, None),
stem.ControllerError(None))
auth_method_combinations = test.get_all_combinations(
[
stem.connection.AuthMethod.NONE,
stem.connection.AuthMethod.PASSWORD,
stem.connection.AuthMethod.COOKIE,
stem.connection.AuthMethod.SAFECOOKIE,
stem.connection.AuthMethod.UNKNOWN,
],
include_empty=True)
protocolinfo = ControlMessage.from_str(
'250-PROTOCOLINFO 1\r\n250 OK\r\n', 'PROTOCOLINFO')
protocolinfo.cookie_path = '/tmp/blah'
for auth_methods in auth_method_combinations:
for auth_none_exc in all_auth_none_exc:
for auth_password_exc in all_auth_password_exc:
for auth_cookie_exc in all_auth_cookie_exc:
# Skip iteration if it's to test exceptions for authentication
# we're not using.
if auth_none_exc and stem.connection.AuthMethod.NONE not in auth_methods:
continue
elif auth_password_exc and stem.connection.AuthMethod.PASSWORD not in auth_methods:
continue
elif auth_cookie_exc and stem.connection.AuthMethod.COOKIE not in auth_methods and stem.connection.AuthMethod.SAFECOOKIE not in auth_methods:
continue
# Determine if the authenticate() call will succeed and mock each
# of the authenticate_* function to raise its given exception.
#
# This implementation is slightly inaccurate in a couple regards...
# a. it raises safecookie exceptions from authenticate_cookie()
# b. exceptions raised by authenticate_cookie() and
# authenticate_safecookie() are always the same
#
# However, adding another loop for safe_cookie exceptions means
# multiplying our runtime many fold. This exercises everything that
# matters so the above inaccuracies seem fine.
expect_success = False
protocolinfo.auth_methods = auth_methods
for auth_method in auth_methods:
if auth_method == stem.connection.AuthMethod.NONE:
auth_mock, raised_exc = authenticate_none_mock, auth_none_exc
elif auth_method == stem.connection.AuthMethod.PASSWORD:
auth_mock, raised_exc = authenticate_password_mock, auth_password_exc
elif auth_method == stem.connection.AuthMethod.COOKIE:
auth_mock, raised_exc = authenticate_cookie_mock, auth_cookie_exc
elif auth_method == stem.connection.AuthMethod.SAFECOOKIE:
auth_mock, raised_exc = authenticate_safecookie_mock, auth_cookie_exc
if raised_exc:
auth_mock.side_effect = raised_exc
else:
auth_mock.side_effect = None
expect_success = True
if expect_success:
stem.connection.authenticate(
None, 'blah', None, protocolinfo)
else:
self.assertRaises(
stem.connection.AuthenticationFailure,
stem.connection.authenticate, None, 'blah',
None, protocolinfo)
# revert logging back to normal
stem_logger.setLevel(log.logging_level(log.TRACE))
# if we have verbose logging then provide the testing config
our_level = stem.util.log.logging_level(CONFIG["argument.log"])
info_level = stem.util.log.logging_level(stem.util.log.INFO)
if our_level <= info_level:
test.output.print_config(test_config)
error_tracker = test.output.ErrorTracker()
output_filters = (
error_tracker.get_filter(),
test.output.strip_module,
test.output.align_results,
test.output.colorize,
)
stem_logger = log.get_logger()
logging_buffer = log.LogBuffer(CONFIG["argument.log"])
stem_logger.addHandler(logging_buffer)
test.output.print_divider("INITIALISING", True)
test.output.print_line("Performing startup activities...", *test.runner.STATUS_ATTR)
_clean_orphaned_pyc()
print
if CONFIG["argument.unit"]:
test.output.print_divider("UNIT TESTS", True)
error_tracker.set_category("UNIT TEST")
for test_class in UNIT_TESTS:
def test_all_use_cases(self):
"""
Does basic validation that all valid use cases for the PROTOCOLINFO input
and dependent functions result in either success or a AuthenticationFailed
subclass being raised.
"""
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# exceptions that the authentication functions are documented to raise
all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None))
all_auth_password_exc = (
None,
stem.connection.PasswordAuthRejected(None),
stem.connection.IncorrectPassword(None))
all_auth_cookie_exc = (
None,
stem.connection.IncorrectCookieSize(None, False, None),
stem.connection.UnreadableCookieFile(None, False, None),
stem.connection.CookieAuthRejected(None, False, None),
stem.connection.IncorrectCookieValue(None, False, None),
stem.connection.UnrecognizedAuthChallengeMethod(None, None, None),
stem.connection.AuthChallengeFailed(None, None),
stem.connection.AuthSecurityFailure(None, None),
stem.connection.InvalidClientNonce(None, None))
# authentication functions might raise a controller error when
# 'suppress_ctl_errors' is False, so including those
control_exc = (
stem.ProtocolError(None),
stem.SocketError(None),
stem.SocketClosed(None))
all_auth_none_exc += control_exc
all_auth_password_exc += control_exc
all_auth_cookie_exc += control_exc
auth_method_combinations = mocking.get_all_combinations([
stem.connection.AuthMethod.NONE,
stem.connection.AuthMethod.PASSWORD,
stem.connection.AuthMethod.COOKIE,
stem.connection.AuthMethod.SAFECOOKIE,
stem.connection.AuthMethod.UNKNOWN,
], include_empty = True)
for protocolinfo_auth_methods in auth_method_combinations:
# protocolinfo input for the authenticate() call we'll be making
protocolinfo_arg = mocking.get_protocolinfo_response(
auth_methods = protocolinfo_auth_methods,
cookie_path = "/tmp/blah",
)
for auth_none_exc in all_auth_none_exc:
for auth_password_exc in all_auth_password_exc:
for auth_cookie_exc in all_auth_cookie_exc:
# Determine if the authenticate() call will succeed and mock each
# of the authenticate_* function to raise its given exception.
#
# This implementation is slightly inaccurate in a couple regards...
# a. it raises safecookie exceptions from authenticate_cookie()
# b. exceptions raised by authenticate_cookie() and
# authenticate_safecookie() are always the same
#
# However, adding another loop for safe_cookie exceptions means
# multiplying our runtime many fold. This exercises everything that
# matters so the above inaccuracies seem fine.
expect_success = False
auth_mocks = {
stem.connection.AuthMethod.NONE:
(stem.connection.authenticate_none, auth_none_exc),
stem.connection.AuthMethod.PASSWORD:
(stem.connection.authenticate_password, auth_password_exc),
stem.connection.AuthMethod.COOKIE:
(stem.connection.authenticate_cookie, auth_cookie_exc),
stem.connection.AuthMethod.SAFECOOKIE:
(stem.connection.authenticate_safecookie, auth_cookie_exc),
}
for auth_method in auth_mocks:
auth_function, raised_exc = auth_mocks[auth_method]
if not raised_exc:
# Mocking this authentication method so it will succeed. If
# it's among the protocolinfo methods then expect success.
mocking.mock(auth_function, mocking.no_op())
expect_success |= auth_method in protocolinfo_auth_methods
else:
mocking.mock(auth_function, mocking.raise_exception(raised_exc))
if expect_success:
stem.connection.authenticate(None, "blah", None, protocolinfo_arg)
else:
self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, "blah", None, protocolinfo_arg)
# revert logging back to normal
stem_logger.setLevel(log.logging_level(log.TRACE))
def __init__(self):
self._tor_lock = True
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
#requests to check the ip
import requests
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
#https://stackoverflow.com/questions/43942689/error-while-receiving-a-control-message-socketclosed-empty-socket-content-i
from stem.util.log import get_logger
logger = get_logger()
logger.propagate = False
# https://stackoverflow.com/questions/45009940/scrapy-with-privoxy-and-tor-how-to-renew-ip/45010141
from toripchanger import TorIpChanger
from stem import Signal
from stem.control import Controller
# password handling
import os
from dotenv import load_dotenv
# TOR
TOR_PASSWORD = os.getenv('TOR_PASS')
# A Tor IP will be reused only after 10 different IPs were used.
ip_changer = TorIpChanger(tor_password=TOR_PASSWORD, reuse_threshold=10)
class ProxyMiddleware(object):
"""
def __init__(self, app):
self.app = app
self.services = dict()
self.tor_process = None
self.socks_port_result = asyncio.Future()
log.get_logger().level = log.logging_level(log.Runlevel.INFO)
def test_all_use_cases(self, authenticate_safecookie_mock, authenticate_cookie_mock, authenticate_password_mock, authenticate_none_mock):
"""
Does basic validation that all valid use cases for the PROTOCOLINFO input
and dependent functions result in either success or a AuthenticationFailed
subclass being raised.
"""
# mute the logger for this test since otherwise the output is overwhelming
stem_logger = log.get_logger()
stem_logger.setLevel(log.logging_level(None))
# exceptions that the authentication functions are documented to raise
all_auth_none_exc = (
None,
stem.connection.OpenAuthRejected(None),
stem.ControllerError(None))
all_auth_password_exc = (
None,
stem.connection.PasswordAuthRejected(None),
stem.connection.IncorrectPassword(None),
stem.ControllerError(None))
all_auth_cookie_exc = (
None,
stem.connection.CookieAuthRejected(None, False, None),
stem.connection.IncorrectCookieValue(None, False, None),
stem.connection.IncorrectCookieSize(None, False, None),
stem.connection.UnreadableCookieFile(None, False, None),
stem.connection.AuthChallengeFailed(None, None),
stem.ControllerError(None))
auth_method_combinations = test.get_all_combinations([
stem.connection.AuthMethod.NONE,
stem.connection.AuthMethod.PASSWORD,
stem.connection.AuthMethod.COOKIE,
stem.connection.AuthMethod.SAFECOOKIE,
stem.connection.AuthMethod.UNKNOWN,
], include_empty = True)
protocolinfo = ControlMessage.from_str('250-PROTOCOLINFO 1\r\n250 OK\r\n', 'PROTOCOLINFO')
protocolinfo.cookie_path = '/tmp/blah'
for auth_methods in auth_method_combinations:
for auth_none_exc in all_auth_none_exc:
for auth_password_exc in all_auth_password_exc:
for auth_cookie_exc in all_auth_cookie_exc:
# Skip iteration if it's to test exceptions for authentication
# we're not using.
if auth_none_exc and stem.connection.AuthMethod.NONE not in auth_methods:
continue
elif auth_password_exc and stem.connection.AuthMethod.PASSWORD not in auth_methods:
continue
elif auth_cookie_exc and stem.connection.AuthMethod.COOKIE not in auth_methods and stem.connection.AuthMethod.SAFECOOKIE not in auth_methods:
continue
# Determine if the authenticate() call will succeed and mock each
# of the authenticate_* function to raise its given exception.
#
# This implementation is slightly inaccurate in a couple regards...
# a. it raises safecookie exceptions from authenticate_cookie()
# b. exceptions raised by authenticate_cookie() and
# authenticate_safecookie() are always the same
#
# However, adding another loop for safe_cookie exceptions means
# multiplying our runtime many fold. This exercises everything that
# matters so the above inaccuracies seem fine.
expect_success = False
protocolinfo.auth_methods = auth_methods
for auth_method in auth_methods:
if auth_method == stem.connection.AuthMethod.NONE:
auth_mock, raised_exc = authenticate_none_mock, auth_none_exc
elif auth_method == stem.connection.AuthMethod.PASSWORD:
auth_mock, raised_exc = authenticate_password_mock, auth_password_exc
elif auth_method == stem.connection.AuthMethod.COOKIE:
auth_mock, raised_exc = authenticate_cookie_mock, auth_cookie_exc
elif auth_method == stem.connection.AuthMethod.SAFECOOKIE:
auth_mock, raised_exc = authenticate_safecookie_mock, auth_cookie_exc
if raised_exc:
auth_mock.side_effect = raised_exc
else:
auth_mock.side_effect = None
expect_success = True
if expect_success:
stem.connection.authenticate(None, 'blah', None, protocolinfo)
else:
self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, 'blah', None, protocolinfo)
# revert logging back to normal
stem_logger.setLevel(log.logging_level(log.TRACE))