示例#1
0
    def __init__(self, config: Config) -> None:
        """
        Initialize Tor Launcher

        :param config: The config class instance that contains global configuration values
        :type config: Config
        :raises TorLauncherInitError: If Tor launcher wasn't able connect to the Tor container
        """
        # Public class attributes
        self.ip_address: str
        self.socks_port: int
        self.control_port: int
        self.relay_fingerprints: List[Any]

        # Private class attributes
        self.__logger = logging.getLogger(__name__)
        self.__config: Config = config
        self.__circuit_id: Controller.new_circuit
        self.__num_retries_on_fail: int = 3
        self.__delay_in_seconds_between_retries: int = 3

        try:
            self.__docker_network_name = self.__config["docker_network"]

        except Exception as exception:
            self.__logger.warning("Could not connect to Tor:\n %s", exception)
            raise TorLauncherInitError from exception

        # Silence the stem logger
        stem_logger = get_logger()
        stem_logger.propagate = False

        # Execute the private methods
        self.__launch_tor_container()
        self.__bind_stem_to_tor_container()
示例#2
0
    def __init__(self):
        self.hostname = Config["proxy"]["hostname"]
        self.socks_port = Config["proxy"]["socks_port"]
        self.ctrl_port = Config["proxy"]["control_port"]
        self.password = Config["proxy"]["password"]

        self.controller = None
        self.ip = ""

        logger = get_logger()
        logger.level = logging.WARNING
示例#3
0
    def test_is_tracing(self):
        logger = log.get_logger()
        original_handlers = logger.handlers
        logger.handlers = [log._NullHandler()]

        try:
            self.assertFalse(log.is_tracing())
            logger.addHandler(log.LogBuffer(log.DEBUG))
            self.assertFalse(log.is_tracing())
            logger.addHandler(log.LogBuffer(log.TRACE))
            self.assertTrue(log.is_tracing())
        finally:
            logger.handlers = original_handlers
示例#4
0
    def test_all_use_cases(self, authenticate_safecookie_mock,
                           authenticate_cookie_mock,
                           authenticate_password_mock, authenticate_none_mock):
        """
    Does basic validation that all valid use cases for the PROTOCOLINFO input
    and dependent functions result in either success or a AuthenticationFailed
    subclass being raised.
    """

        # mute the logger for this test since otherwise the output is overwhelming

        stem_logger = log.get_logger()
        stem_logger.setLevel(log.logging_level(None))

        # exceptions that the authentication functions are documented to raise

        all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None))

        all_auth_password_exc = (None,
                                 stem.connection.PasswordAuthRejected(None),
                                 stem.connection.IncorrectPassword(None))

        all_auth_cookie_exc = (
            None, stem.connection.IncorrectCookieSize(None, False, None),
            stem.connection.UnreadableCookieFile(None, False, None),
            stem.connection.CookieAuthRejected(None, False, None),
            stem.connection.IncorrectCookieValue(None, False, None),
            stem.connection.UnrecognizedAuthChallengeMethod(None, None, None),
            stem.connection.AuthChallengeFailed(None, None),
            stem.connection.AuthSecurityFailure(None, None),
            stem.connection.InvalidClientNonce(None, None))

        # authentication functions might raise a controller error when
        # 'suppress_ctl_errors' is False, so including those

        control_exc = (stem.ProtocolError(None), stem.SocketError(None),
                       stem.SocketClosed(None))

        all_auth_none_exc += control_exc
        all_auth_password_exc += control_exc
        all_auth_cookie_exc += control_exc

        auth_method_combinations = mocking.get_all_combinations(
            [
                stem.connection.AuthMethod.NONE,
                stem.connection.AuthMethod.PASSWORD,
                stem.connection.AuthMethod.COOKIE,
                stem.connection.AuthMethod.SAFECOOKIE,
                stem.connection.AuthMethod.UNKNOWN,
            ],
            include_empty=True)

        for protocolinfo_auth_methods in auth_method_combinations:
            # protocolinfo input for the authenticate() call we'll be making
            protocolinfo_arg = mocking.get_protocolinfo_response(
                auth_methods=protocolinfo_auth_methods,
                cookie_path='/tmp/blah',
            )

            for auth_none_exc in all_auth_none_exc:
                for auth_password_exc in all_auth_password_exc:
                    for auth_cookie_exc in all_auth_cookie_exc:
                        # Determine if the authenticate() call will succeed and mock each
                        # of the authenticate_* function to raise its given exception.
                        #
                        # This implementation is slightly inaccurate in a couple regards...
                        # a. it raises safecookie exceptions from authenticate_cookie()
                        # b. exceptions raised by authenticate_cookie() and
                        #    authenticate_safecookie() are always the same
                        #
                        # However, adding another loop for safe_cookie exceptions means
                        # multiplying our runtime many fold. This exercises everything that
                        # matters so the above inaccuracies seem fine.

                        expect_success = False
                        auth_mocks = {
                            stem.connection.AuthMethod.NONE:
                            (authenticate_none_mock, auth_none_exc),
                            stem.connection.AuthMethod.PASSWORD:
                            (authenticate_password_mock, auth_password_exc),
                            stem.connection.AuthMethod.COOKIE:
                            (authenticate_cookie_mock, auth_cookie_exc),
                            stem.connection.AuthMethod.SAFECOOKIE:
                            (authenticate_safecookie_mock, auth_cookie_exc),
                        }

                        for auth_method in auth_mocks:
                            auth_mock, raised_exc = auth_mocks[auth_method]

                            if not raised_exc:
                                # Mocking this authentication method so it will succeed. If
                                # it's among the protocolinfo methods then expect success.

                                auth_mock.side_effect = None
                                expect_success |= auth_method in protocolinfo_auth_methods
                            else:
                                auth_mock.side_effect = raised_exc

                        if expect_success:
                            stem.connection.authenticate(
                                None, 'blah', None, protocolinfo_arg)
                        else:
                            self.assertRaises(
                                stem.connection.AuthenticationFailure,
                                stem.connection.authenticate, None, 'blah',
                                None, protocolinfo_arg)

        # revert logging back to normal
        stem_logger.setLevel(log.logging_level(log.TRACE))
示例#5
0
 def test_all_use_cases(self):
   """
   Does basic validation that all valid use cases for the PROTOCOLINFO input
   and dependent functions result in either success or a AuthenticationFailed
   subclass being raised.
   """
   
   # mute the logger for this test since otherwise the output is overwhelming
   
   stem_logger = log.get_logger()
   stem_logger.setLevel(log.logging_level(None))
   
   # exceptions that the authentication functions are documented to raise
   all_auth_none_exc = (None,
     stem.connection.OpenAuthRejected(None))
   
   all_auth_password_exc = (None,
     stem.connection.PasswordAuthRejected(None),
     stem.connection.IncorrectPassword(None))
   
   all_auth_cookie_exc = (None,
     stem.connection.IncorrectCookieSize(None, None),
     stem.connection.UnreadableCookieFile(None, None),
     stem.connection.CookieAuthRejected(None, None),
     stem.connection.IncorrectCookieValue(None, None))
   
   # authentication functions might raise a controller error when
   # 'suppress_ctl_errors' is False, so including those
   
   control_exc = (
     stem.socket.ProtocolError(None),
     stem.socket.SocketError(None),
     stem.socket.SocketClosed(None))
   
   all_auth_none_exc += control_exc
   all_auth_password_exc += control_exc
   all_auth_cookie_exc += control_exc
   
   for protocolinfo_auth_methods in _get_all_auth_method_combinations():
     # protocolinfo input for the authenticate() call we'll be making
     protocolinfo_arg = mocking.get_protocolinfo_response(
       auth_methods = protocolinfo_auth_methods,
       cookie_path = "/tmp/blah",
     )
     
     for auth_none_exc in all_auth_none_exc:
       for auth_password_exc in all_auth_password_exc:
         for auth_cookie_exc in all_auth_cookie_exc:
           # determine if the authenticate() call will succeed and mock each
           # of the authenticate_* function to raise its given exception
           
           expect_success = False
           auth_mocks = {
             stem.connection.AuthMethod.NONE:
               (stem.connection.authenticate_none, auth_none_exc),
             stem.connection.AuthMethod.PASSWORD:
               (stem.connection.authenticate_password, auth_password_exc),
             stem.connection.AuthMethod.COOKIE:
               (stem.connection.authenticate_cookie, auth_cookie_exc),
           }
           
           for auth_method in auth_mocks:
             auth_function, raised_exc = auth_mocks[auth_method]
             
             if not raised_exc:
               # Mocking this authentication method so it will succeed. If
               # it's among the protocolinfo methods then expect success.
               
               mocking.mock(auth_function, mocking.no_op())
               expect_success |= auth_method in protocolinfo_auth_methods
             else:
               mocking.mock(auth_function, mocking.raise_exception(raised_exc))
           
           if expect_success:
             stem.connection.authenticate(None, "blah", None, protocolinfo_arg)
           else:
             self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, "blah", None, protocolinfo_arg)
   
   # revert logging back to normal
   stem_logger.setLevel(log.logging_level(log.TRACE))
示例#6
0
    def test_all_use_cases(self, authenticate_safecookie_mock,
                           authenticate_cookie_mock,
                           authenticate_password_mock, authenticate_none_mock):
        """
    Does basic validation that all valid use cases for the PROTOCOLINFO input
    and dependent functions result in either success or a AuthenticationFailed
    subclass being raised.
    """

        # mute the logger for this test since otherwise the output is overwhelming

        stem_logger = log.get_logger()
        stem_logger.setLevel(log.logging_level(None))

        # exceptions that the authentication functions are documented to raise

        all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None),
                             stem.ControllerError(None))

        all_auth_password_exc = (None,
                                 stem.connection.PasswordAuthRejected(None),
                                 stem.connection.IncorrectPassword(None),
                                 stem.ControllerError(None))

        all_auth_cookie_exc = (None,
                               stem.connection.CookieAuthRejected(
                                   None, False, None),
                               stem.connection.IncorrectCookieValue(
                                   None, False, None),
                               stem.connection.IncorrectCookieSize(
                                   None, False, None),
                               stem.connection.UnreadableCookieFile(
                                   None, False, None),
                               stem.connection.AuthChallengeFailed(None, None),
                               stem.ControllerError(None))

        auth_method_combinations = test.get_all_combinations(
            [
                stem.connection.AuthMethod.NONE,
                stem.connection.AuthMethod.PASSWORD,
                stem.connection.AuthMethod.COOKIE,
                stem.connection.AuthMethod.SAFECOOKIE,
                stem.connection.AuthMethod.UNKNOWN,
            ],
            include_empty=True)

        protocolinfo = ControlMessage.from_str(
            '250-PROTOCOLINFO 1\r\n250 OK\r\n', 'PROTOCOLINFO')
        protocolinfo.cookie_path = '/tmp/blah'

        for auth_methods in auth_method_combinations:
            for auth_none_exc in all_auth_none_exc:
                for auth_password_exc in all_auth_password_exc:
                    for auth_cookie_exc in all_auth_cookie_exc:
                        # Skip iteration if it's to test exceptions for authentication
                        # we're not using.

                        if auth_none_exc and stem.connection.AuthMethod.NONE not in auth_methods:
                            continue
                        elif auth_password_exc and stem.connection.AuthMethod.PASSWORD not in auth_methods:
                            continue
                        elif auth_cookie_exc and stem.connection.AuthMethod.COOKIE not in auth_methods and stem.connection.AuthMethod.SAFECOOKIE not in auth_methods:
                            continue

                        # Determine if the authenticate() call will succeed and mock each
                        # of the authenticate_* function to raise its given exception.
                        #
                        # This implementation is slightly inaccurate in a couple regards...
                        # a. it raises safecookie exceptions from authenticate_cookie()
                        # b. exceptions raised by authenticate_cookie() and
                        #    authenticate_safecookie() are always the same
                        #
                        # However, adding another loop for safe_cookie exceptions means
                        # multiplying our runtime many fold. This exercises everything that
                        # matters so the above inaccuracies seem fine.

                        expect_success = False
                        protocolinfo.auth_methods = auth_methods

                        for auth_method in auth_methods:
                            if auth_method == stem.connection.AuthMethod.NONE:
                                auth_mock, raised_exc = authenticate_none_mock, auth_none_exc
                            elif auth_method == stem.connection.AuthMethod.PASSWORD:
                                auth_mock, raised_exc = authenticate_password_mock, auth_password_exc
                            elif auth_method == stem.connection.AuthMethod.COOKIE:
                                auth_mock, raised_exc = authenticate_cookie_mock, auth_cookie_exc
                            elif auth_method == stem.connection.AuthMethod.SAFECOOKIE:
                                auth_mock, raised_exc = authenticate_safecookie_mock, auth_cookie_exc

                            if raised_exc:
                                auth_mock.side_effect = raised_exc
                            else:
                                auth_mock.side_effect = None
                                expect_success = True

                        if expect_success:
                            stem.connection.authenticate(
                                None, 'blah', None, protocolinfo)
                        else:
                            self.assertRaises(
                                stem.connection.AuthenticationFailure,
                                stem.connection.authenticate, None, 'blah',
                                None, protocolinfo)

        # revert logging back to normal
        stem_logger.setLevel(log.logging_level(log.TRACE))
示例#7
0
  # if we have verbose logging then provide the testing config
  our_level = stem.util.log.logging_level(CONFIG["argument.log"])
  info_level = stem.util.log.logging_level(stem.util.log.INFO)

  if our_level <= info_level:
    test.output.print_config(test_config)

  error_tracker = test.output.ErrorTracker()
  output_filters = (
    error_tracker.get_filter(),
    test.output.strip_module,
    test.output.align_results,
    test.output.colorize,
  )

  stem_logger = log.get_logger()
  logging_buffer = log.LogBuffer(CONFIG["argument.log"])
  stem_logger.addHandler(logging_buffer)

  test.output.print_divider("INITIALISING", True)

  test.output.print_line("Performing startup activities...", *test.runner.STATUS_ATTR)
  _clean_orphaned_pyc()

  print

  if CONFIG["argument.unit"]:
    test.output.print_divider("UNIT TESTS", True)
    error_tracker.set_category("UNIT TEST")

    for test_class in UNIT_TESTS:
示例#8
0
  def test_all_use_cases(self):
    """
    Does basic validation that all valid use cases for the PROTOCOLINFO input
    and dependent functions result in either success or a AuthenticationFailed
    subclass being raised.
    """

    # mute the logger for this test since otherwise the output is overwhelming

    stem_logger = log.get_logger()
    stem_logger.setLevel(log.logging_level(None))

    # exceptions that the authentication functions are documented to raise

    all_auth_none_exc = (None, stem.connection.OpenAuthRejected(None))

    all_auth_password_exc = (
      None,
      stem.connection.PasswordAuthRejected(None),
      stem.connection.IncorrectPassword(None))

    all_auth_cookie_exc = (
      None,
      stem.connection.IncorrectCookieSize(None, False, None),
      stem.connection.UnreadableCookieFile(None, False, None),
      stem.connection.CookieAuthRejected(None, False, None),
      stem.connection.IncorrectCookieValue(None, False, None),
      stem.connection.UnrecognizedAuthChallengeMethod(None, None, None),
      stem.connection.AuthChallengeFailed(None, None),
      stem.connection.AuthSecurityFailure(None, None),
      stem.connection.InvalidClientNonce(None, None))

    # authentication functions might raise a controller error when
    # 'suppress_ctl_errors' is False, so including those

    control_exc = (
      stem.ProtocolError(None),
      stem.SocketError(None),
      stem.SocketClosed(None))

    all_auth_none_exc += control_exc
    all_auth_password_exc += control_exc
    all_auth_cookie_exc += control_exc

    auth_method_combinations = mocking.get_all_combinations([
      stem.connection.AuthMethod.NONE,
      stem.connection.AuthMethod.PASSWORD,
      stem.connection.AuthMethod.COOKIE,
      stem.connection.AuthMethod.SAFECOOKIE,
      stem.connection.AuthMethod.UNKNOWN,
    ], include_empty = True)

    for protocolinfo_auth_methods in auth_method_combinations:
      # protocolinfo input for the authenticate() call we'll be making
      protocolinfo_arg = mocking.get_protocolinfo_response(
        auth_methods = protocolinfo_auth_methods,
        cookie_path = "/tmp/blah",
      )

      for auth_none_exc in all_auth_none_exc:
        for auth_password_exc in all_auth_password_exc:
          for auth_cookie_exc in all_auth_cookie_exc:
            # Determine if the authenticate() call will succeed and mock each
            # of the authenticate_* function to raise its given exception.
            #
            # This implementation is slightly inaccurate in a couple regards...
            # a. it raises safecookie exceptions from authenticate_cookie()
            # b. exceptions raised by authenticate_cookie() and
            #    authenticate_safecookie() are always the same
            #
            # However, adding another loop for safe_cookie exceptions means
            # multiplying our runtime many fold. This exercises everything that
            # matters so the above inaccuracies seem fine.

            expect_success = False
            auth_mocks = {
              stem.connection.AuthMethod.NONE:
                (stem.connection.authenticate_none, auth_none_exc),
              stem.connection.AuthMethod.PASSWORD:
                (stem.connection.authenticate_password, auth_password_exc),
              stem.connection.AuthMethod.COOKIE:
                (stem.connection.authenticate_cookie, auth_cookie_exc),
              stem.connection.AuthMethod.SAFECOOKIE:
                (stem.connection.authenticate_safecookie, auth_cookie_exc),
            }

            for auth_method in auth_mocks:
              auth_function, raised_exc = auth_mocks[auth_method]

              if not raised_exc:
                # Mocking this authentication method so it will succeed. If
                # it's among the protocolinfo methods then expect success.

                mocking.mock(auth_function, mocking.no_op())
                expect_success |= auth_method in protocolinfo_auth_methods
              else:
                mocking.mock(auth_function, mocking.raise_exception(raised_exc))

            if expect_success:
              stem.connection.authenticate(None, "blah", None, protocolinfo_arg)
            else:
              self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, "blah", None, protocolinfo_arg)

    # revert logging back to normal
    stem_logger.setLevel(log.logging_level(log.TRACE))
示例#9
0
 def __init__(self):
     self._tor_lock = True
     
     # mute the logger for this test since otherwise the output is overwhelming
     stem_logger = log.get_logger()
     stem_logger.setLevel(log.logging_level(None))
示例#10
0
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
#requests to check the ip
import requests
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
#https://stackoverflow.com/questions/43942689/error-while-receiving-a-control-message-socketclosed-empty-socket-content-i
from stem.util.log import get_logger
logger = get_logger()
logger.propagate = False

# https://stackoverflow.com/questions/45009940/scrapy-with-privoxy-and-tor-how-to-renew-ip/45010141
from toripchanger import TorIpChanger
from stem import Signal
from stem.control import Controller
# password handling
import os
from dotenv import load_dotenv
# TOR
TOR_PASSWORD = os.getenv('TOR_PASS')
# A Tor IP will be reused only after 10 different IPs were used.
ip_changer = TorIpChanger(tor_password=TOR_PASSWORD, reuse_threshold=10)


class ProxyMiddleware(object):
    """
示例#11
0
 def __init__(self, app):
     self.app = app
     self.services = dict()
     self.tor_process = None
     self.socks_port_result = asyncio.Future()
     log.get_logger().level = log.logging_level(log.Runlevel.INFO)
示例#12
0
  def test_all_use_cases(self, authenticate_safecookie_mock, authenticate_cookie_mock, authenticate_password_mock, authenticate_none_mock):
    """
    Does basic validation that all valid use cases for the PROTOCOLINFO input
    and dependent functions result in either success or a AuthenticationFailed
    subclass being raised.
    """

    # mute the logger for this test since otherwise the output is overwhelming

    stem_logger = log.get_logger()
    stem_logger.setLevel(log.logging_level(None))

    # exceptions that the authentication functions are documented to raise

    all_auth_none_exc = (
      None,
      stem.connection.OpenAuthRejected(None),
      stem.ControllerError(None))

    all_auth_password_exc = (
      None,
      stem.connection.PasswordAuthRejected(None),
      stem.connection.IncorrectPassword(None),
      stem.ControllerError(None))

    all_auth_cookie_exc = (
      None,
      stem.connection.CookieAuthRejected(None, False, None),
      stem.connection.IncorrectCookieValue(None, False, None),
      stem.connection.IncorrectCookieSize(None, False, None),
      stem.connection.UnreadableCookieFile(None, False, None),
      stem.connection.AuthChallengeFailed(None, None),
      stem.ControllerError(None))

    auth_method_combinations = test.get_all_combinations([
      stem.connection.AuthMethod.NONE,
      stem.connection.AuthMethod.PASSWORD,
      stem.connection.AuthMethod.COOKIE,
      stem.connection.AuthMethod.SAFECOOKIE,
      stem.connection.AuthMethod.UNKNOWN,
    ], include_empty = True)

    protocolinfo = ControlMessage.from_str('250-PROTOCOLINFO 1\r\n250 OK\r\n', 'PROTOCOLINFO')
    protocolinfo.cookie_path = '/tmp/blah'

    for auth_methods in auth_method_combinations:
      for auth_none_exc in all_auth_none_exc:
        for auth_password_exc in all_auth_password_exc:
          for auth_cookie_exc in all_auth_cookie_exc:
            # Skip iteration if it's to test exceptions for authentication
            # we're not using.

            if auth_none_exc and stem.connection.AuthMethod.NONE not in auth_methods:
              continue
            elif auth_password_exc and stem.connection.AuthMethod.PASSWORD not in auth_methods:
              continue
            elif auth_cookie_exc and stem.connection.AuthMethod.COOKIE not in auth_methods and stem.connection.AuthMethod.SAFECOOKIE not in auth_methods:
              continue

            # Determine if the authenticate() call will succeed and mock each
            # of the authenticate_* function to raise its given exception.
            #
            # This implementation is slightly inaccurate in a couple regards...
            # a. it raises safecookie exceptions from authenticate_cookie()
            # b. exceptions raised by authenticate_cookie() and
            #    authenticate_safecookie() are always the same
            #
            # However, adding another loop for safe_cookie exceptions means
            # multiplying our runtime many fold. This exercises everything that
            # matters so the above inaccuracies seem fine.

            expect_success = False
            protocolinfo.auth_methods = auth_methods

            for auth_method in auth_methods:
              if auth_method == stem.connection.AuthMethod.NONE:
                auth_mock, raised_exc = authenticate_none_mock, auth_none_exc
              elif auth_method == stem.connection.AuthMethod.PASSWORD:
                auth_mock, raised_exc = authenticate_password_mock, auth_password_exc
              elif auth_method == stem.connection.AuthMethod.COOKIE:
                auth_mock, raised_exc = authenticate_cookie_mock, auth_cookie_exc
              elif auth_method == stem.connection.AuthMethod.SAFECOOKIE:
                auth_mock, raised_exc = authenticate_safecookie_mock, auth_cookie_exc

              if raised_exc:
                auth_mock.side_effect = raised_exc
              else:
                auth_mock.side_effect = None
                expect_success = True

            if expect_success:
              stem.connection.authenticate(None, 'blah', None, protocolinfo)
            else:
              self.assertRaises(stem.connection.AuthenticationFailure, stem.connection.authenticate, None, 'blah', None, protocolinfo)

    # revert logging back to normal
    stem_logger.setLevel(log.logging_level(log.TRACE))