def __init__(self, master, slaves=[], document_class=dict, tz_aware=False): """Create a new Master-Slave connection. The resultant connection should be interacted with using the same mechanisms as a regular `MongoClient`. The `MongoClient` instances used to create this `MasterSlaveConnection` can themselves make use of connection pooling, etc. `MongoClient` instances used as slaves should be created with the read_preference option set to :attr:`~pymongo.read_preferences.ReadPreference.SECONDARY`. Write concerns are inherited from `master` and can be changed in this instance. Raises TypeError if `master` is not an instance of `MongoClient` or slaves is not a list of at least one `MongoClient` instances. :Parameters: - `master`: `MongoClient` instance for the writable Master - `slaves`: list of `MongoClient` instances for the read-only slaves - `document_class` (optional): default class to use for documents returned from queries on this connection - `tz_aware` (optional): if ``True``, :class:`~datetime.datetime` instances returned as values in a document by this :class:`MasterSlaveConnection` will be timezone aware (otherwise they will be naive) """ if not isinstance(master, MongoClient): raise TypeError("master must be a MongoClient instance") if not isinstance(slaves, list) or len(slaves) == 0: raise TypeError("slaves must be a list of length >= 1") for slave in slaves: if not isinstance(slave, MongoClient): raise TypeError("slave %r is not an instance of MongoClient" % slave) warnings.warn( "MasterSlaveConnection is deprecated, and will be" " removed in PyMongo 3.0.", DeprecationWarning, stacklevel=2) validate_boolean('tz_aware', tz_aware) codec_options = CodecOptions(document_class, tz_aware) super(MasterSlaveConnection, self).__init__(read_preference=ReadPreference.SECONDARY, safe=master.safe, codec_options=codec_options, **master.write_concern) self.__master = master self.__slaves = slaves self.__request_counter = thread_util.Counter(master.use_greenlets)
def _test_counter(self, use_greenlets): counter = thread_util.Counter(use_greenlets) self.assertEqual(0, counter.dec()) self.assertEqual(0, counter.get()) self.assertEqual(0, counter.dec()) self.assertEqual(0, counter.get()) done = set() def f(n): for i in xrange(n): self.assertEqual(i, counter.get()) self.assertEqual(i + 1, counter.inc()) for i in xrange(n, 0, -1): self.assertEqual(i, counter.get()) self.assertEqual(i - 1, counter.dec()) self.assertEqual(0, counter.get()) # Extra decrements have no effect self.assertEqual(0, counter.dec()) self.assertEqual(0, counter.get()) self.assertEqual(0, counter.dec()) self.assertEqual(0, counter.get()) done.add(n) if use_greenlets: greenlets = [ greenlet.greenlet(my_partial(f, i)) for i in xrange(10) ] looplet(greenlets) else: threads = [ threading.Thread(target=my_partial(f, i)) for i in xrange(10) ] for t in threads: t.start() for t in threads: t.join() self.assertEqual(10, len(done))
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl, use_greenlets, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None, wait_queue_timeout=None, wait_queue_multiple=None): """ :Parameters: - `pair`: a (hostname, port) tuple - `max_size`: The maximum number of open sockets. Calls to `get_socket` will block if this is set, this pool has opened `max_size` sockets, and there are none idle. Set to `None` to disable. - `net_timeout`: timeout in seconds for operations on open connection - `conn_timeout`: timeout in seconds for establishing connection - `use_ssl`: bool, if True use an encrypted connection - `use_greenlets`: bool, if True then start_request() assigns a socket to the current greenlet - otherwise it is assigned to the current thread - `ssl_keyfile`: The private keyfile used to identify the local connection against mongod. If included with the ``certfile` then only the ``ssl_certfile`` is needed. Implies ``ssl=True``. - `ssl_certfile`: The certificate file used to identify the local connection against mongod. Implies ``ssl=True``. - `ssl_cert_reqs`: Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. It must be one of the three values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` (not required, but validated if provided), or ``ssl.CERT_REQUIRED`` (required and validated). If the value of this parameter is not ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point to a file of CA certificates. Implies ``ssl=True``. - `ssl_ca_certs`: The ca_certs file contains a set of concatenated "certification authority" certificates, which are used to validate certificates passed from the other end of the connection. Implies ``ssl=True``. - `wait_queue_timeout`: (integer) How long (in seconds) a thread will wait for a socket from the pool if the pool has no free sockets. - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give the number of threads allowed to wait for a socket at one time. """ # Only check a socket's health with _closed() every once in a while. # Can override for testing: 0 to always check, None to never check. self._check_interval_seconds = 1 self.sockets = set() self.lock = threading.Lock() # Keep track of resets, so we notice sockets created before the most # recent reset and close them. self.pool_id = 0 self.pid = os.getpid() self.pair = pair self.max_size = max_size self.net_timeout = net_timeout self.conn_timeout = conn_timeout self.wait_queue_timeout = wait_queue_timeout self.wait_queue_multiple = wait_queue_multiple self.use_ssl = use_ssl self.ssl_keyfile = ssl_keyfile self.ssl_certfile = ssl_certfile self.ssl_cert_reqs = ssl_cert_reqs self.ssl_ca_certs = ssl_ca_certs if HAS_SSL and use_ssl and not ssl_cert_reqs: self.ssl_cert_reqs = ssl.CERT_NONE # Map self._ident.get() -> request socket self._tid_to_sock = {} if use_greenlets and not thread_util.have_gevent: raise ConfigurationError("The Gevent module is not available. " "Install the gevent package from PyPI.") self._ident = thread_util.create_ident(use_greenlets) # Count the number of calls to start_request() per thread or greenlet self._request_counter = thread_util.Counter(use_greenlets) if self.wait_queue_multiple is None or self.max_size is None: max_waiters = None else: max_waiters = self.max_size * self.wait_queue_multiple self._socket_semaphore = thread_util.create_semaphore( self.max_size, max_waiters, use_greenlets)
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl, use_greenlets, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None, wait_queue_timeout=None, wait_queue_multiple=None, socket_keepalive=False, ssl_match_hostname=True): """ :Parameters: - `pair`: a (hostname, port) tuple - `max_size`: The maximum number of open sockets. Calls to `get_socket` will block if this is set, this pool has opened `max_size` sockets, and there are none idle. Set to `None` to disable. - `net_timeout`: timeout in seconds for operations on open connection - `conn_timeout`: timeout in seconds for establishing connection - `use_ssl`: bool, if True use an encrypted connection - `use_greenlets`: bool, if True then start_request() assigns a socket to the current greenlet - otherwise it is assigned to the current thread - `ssl_keyfile`: The private keyfile used to identify the local connection against mongod. If included with the ``certfile` then only the ``ssl_certfile`` is needed. Implies ``ssl=True``. - `ssl_certfile`: The certificate file used to identify the local connection against mongod. Implies ``ssl=True``. - `ssl_cert_reqs`: Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. It must be one of the three values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` (not required, but validated if provided), or ``ssl.CERT_REQUIRED`` (required and validated). If the value of this parameter is not ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point to a file of CA certificates. Implies ``ssl=True``. - `ssl_ca_certs`: The ca_certs file contains a set of concatenated "certification authority" certificates, which are used to validate certificates passed from the other end of the connection. Implies ``ssl=True``. - `wait_queue_timeout`: (integer) How long (in seconds) a thread will wait for a socket from the pool if the pool has no free sockets. - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give the number of threads allowed to wait for a socket at one time. - `socket_keepalive`: (boolean) Whether to send periodic keep-alive packets on connected sockets. Defaults to ``False`` (do not send keep-alive packets). - `ssl_match_hostname`: If ``True`` (the default), and `ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname verification using the :func:`~ssl.match_hostname` function from python's :mod:`~ssl` module. Think very carefully before setting this to ``False`` as that could make your application vulnerable to man-in-the-middle attacks. """ # Only check a socket's health with _closed() every once in a while. # Can override for testing: 0 to always check, None to never check. self._check_interval_seconds = 1 self.sockets = set() self.lock = threading.Lock() # Keep track of resets, so we notice sockets created before the most # recent reset and close them. self.pool_id = 0 self.pid = os.getpid() self.pair = pair self.max_size = max_size self.net_timeout = net_timeout self.conn_timeout = conn_timeout self.wait_queue_timeout = wait_queue_timeout self.wait_queue_multiple = wait_queue_multiple self.socket_keepalive = socket_keepalive self.ssl_ctx = None self.ssl_match_hostname = ssl_match_hostname if HAS_SSL and use_ssl: # PROTOCOL_TLS_CLIENT added and PROTOCOL_SSLv23 # deprecated in CPython 3.6 self.ssl_ctx = _SSLContext( getattr(ssl, 'PROTOCOL_TLS_CLIENT', ssl.PROTOCOL_SSLv23)) # SSLContext.check_hostname was added in 2.7.9 and 3.4. Using it # forces the use of SNI, which PyMongo 2.x doesn't support. # PROTOCOL_TLS_CLIENT enables this by default. Since we call # match_hostname directly disable this explicitly. if hasattr(self.ssl_ctx, "check_hostname"): self.ssl_ctx.check_hostname = False # load_cert_chain and load_verify_locations can fail # if the file contents are invalid. if ssl_certfile is not None: try: self.ssl_ctx.load_cert_chain(ssl_certfile, ssl_keyfile) except ssl.SSLError: pass if ssl_ca_certs is not None: try: self.ssl_ctx.load_verify_locations(ssl_ca_certs) except ssl.SSLError: pass # PROTOCOL_TLS_CLIENT sets verify_mode to CERT_REQUIRED so # we always have to set this explicitly. if ssl_cert_reqs is not None: self.ssl_ctx.verify_mode = ssl_cert_reqs else: self.ssl_ctx.verify_mode = ssl.CERT_NONE # Map self._ident.get() -> request socket self._tid_to_sock = {} if use_greenlets and not thread_util.have_gevent: raise ConfigurationError("The Gevent module is not available. " "Install the gevent package from PyPI.") self._ident = thread_util.create_ident(use_greenlets) # Count the number of calls to start_request() per thread or greenlet self._request_counter = thread_util.Counter(use_greenlets) if self.wait_queue_multiple is None or self.max_size is None: max_waiters = None else: max_waiters = self.max_size * self.wait_queue_multiple self._socket_semaphore = thread_util.create_semaphore( self.max_size, max_waiters, use_greenlets)
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl, use_greenlets, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None): """ :Parameters: - `pair`: a (hostname, port) tuple - `max_size`: approximate number of idle connections to keep open - `net_timeout`: timeout in seconds for operations on open connection - `conn_timeout`: timeout in seconds for establishing connection - `use_ssl`: bool, if True use an encrypted connection - `use_greenlets`: bool, if True then start_request() assigns a socket to the current greenlet - otherwise it is assigned to the current thread - `ssl_keyfile`: The private keyfile used to identify the local connection against mongod. If included with the ``certfile` then only the ``ssl_certfile`` is needed. Implies ``ssl=True``. - `ssl_certfile`: The certificate file used to identify the local connection against mongod. Implies ``ssl=True``. - `ssl_cert_reqs`: Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. It must be one of the three values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` (not required, but validated if provided), or ``ssl.CERT_REQUIRED`` (required and validated). If the value of this parameter is not ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point to a file of CA certificates. Implies ``ssl=True``. - `ssl_ca_certs`: The ca_certs file contains a set of concatenated "certification authority" certificates, which are used to validate certificates passed from the other end of the connection. Implies ``ssl=True``. """ if use_greenlets and not thread_util.have_greenlet: raise ConfigurationError( "The greenlet module is not available. " "Install the greenlet package from PyPI." ) self.sockets = set() self.lock = threading.Lock() # Keep track of resets, so we notice sockets created before the most # recent reset and close them. self.pool_id = 0 self.pid = os.getpid() self.pair = pair self.max_size = max_size self.net_timeout = net_timeout self.conn_timeout = conn_timeout self.use_ssl = use_ssl self.ssl_keyfile = ssl_keyfile self.ssl_certfile = ssl_certfile self.ssl_cert_reqs = ssl_cert_reqs self.ssl_ca_certs = ssl_ca_certs if HAS_SSL and use_ssl and not ssl_cert_reqs: self.ssl_cert_reqs = ssl.CERT_NONE self._ident = thread_util.create_ident(use_greenlets) # Map self._ident.get() -> request socket self._tid_to_sock = {} # Count the number of calls to start_request() per thread or greenlet self._request_counter = thread_util.Counter(use_greenlets)