def _send_message_with_response(self, msg, _connection_to_use=None, _must_use_master=False, **kwargs): """Send a message to Mongo and return the response. Sends the given message and returns the response. :Parameters: - `msg`: (request_id, data) pair making up the message to send """ read_pref = kwargs.get("read_preference", ReadPreference.PRIMARY) mongo = None try: if _connection_to_use is not None: if _connection_to_use == -1: mongo = self.__find_primary() else: mongo = self.__pools[_connection_to_use] return mongo["pool"].host, self.__send_and_receive(mongo, msg, **kwargs) elif _must_use_master or not read_pref: mongo = self.__find_primary() return mongo["pool"].host, self.__send_and_receive(mongo, msg, **kwargs) except AutoReconnect: if mongo == self.__writer: self.disconnect() raise errors = [] for host in helpers.shuffled(self.__readers): try: mongo = self.__pools[host] return host, self.__send_and_receive(mongo, msg, **kwargs) except AutoReconnect, why: errors.append(str(why))
def _send_message_with_response(self, message, _connection_to_use=None, _must_use_master=False, **kwargs): """Receive a message from Mongo. Sends the given message and returns a (connection_id, response) pair. :Parameters: - `operation`: opcode of the message to send - `data`: data to send """ if _connection_to_use is not None: if _connection_to_use == -1: return (-1, self.__master._send_message_with_response(message, **kwargs)) else: return ( _connection_to_use, self.__slaves[_connection_to_use]._send_message_with_response(message, **kwargs), ) # _must_use_master is set for commands, which must be sent to the # master instance. any queries in a request must be sent to the # master since that is where writes go. if _must_use_master or self.__in_request: return (-1, self.__master._send_message_with_response(message, **kwargs)) # Iterate through the slaves randomly until we have success. Raise # reconnect if they all fail. for connection_id in helpers.shuffled(range(len(self.__slaves))): try: slave = self.__slaves[connection_id] return (connection_id, slave._send_message_with_response(message, **kwargs)) except AutoReconnect: pass raise AutoReconnect("failed to connect to slaves")
def kooaba_upload(count): from kooaba.bridge import kooaba_bridge from pymongo.helpers import shuffled logger = logging.getLogger(__name__) logger.info("Kooaba uploading " + str(count) + " images") images = shuffled(image_collection.find()) for i in range(count): image = images[i] logger.info("Start uploading " + str(image)) kooaba_bridge.upload_image(image)
def _send_message_with_response(self, message, _connection_to_use=None, _must_use_master=False, **kwargs): """Receive a message from Mongo. Sends the given message and returns a (connection_id, response) pair. :Parameters: - `operation`: opcode of the message to send - `data`: data to send """ if _connection_to_use is not None: if _connection_to_use == -1: member = self.__master conn = -1 else: member = self.__slaves[_connection_to_use] conn = _connection_to_use return (conn, member._send_message_with_response(message, **kwargs)[1]) # _must_use_master is set for commands, which must be sent to the # master instance. any queries in a request must be sent to the # master since that is where writes go. if _must_use_master or self.in_request(): return (-1, self.__master._send_message_with_response( message, **kwargs)[1]) # Iterate through the slaves randomly until we have success. Raise # reconnect if they all fail. for connection_id in helpers.shuffled(xrange(len(self.__slaves))): try: slave = self.__slaves[connection_id] return (connection_id, slave._send_message_with_response(message, **kwargs)[1]) except AutoReconnect: pass raise AutoReconnect("failed to connect to slaves")
def _send_message_with_response(self, msg, _connection_to_use=None, _must_use_master=False, **kwargs): """Send a message to Mongo and return the response. Sends the given message and returns the response. :Parameters: - `msg`: (request_id, data) pair making up the message to send """ read_pref = kwargs.get('read_preference', ReadPreference.PRIMARY) mongo = None try: if _connection_to_use is not None: if _connection_to_use == -1: mongo = self.__find_primary() else: mongo = self.__pools[_connection_to_use] return mongo['pool'].pair, self.__send_and_receive( mongo, msg, **kwargs) elif _must_use_master or not read_pref: mongo = self.__find_primary() return mongo['pool'].pair, self.__send_and_receive( mongo, msg, **kwargs) except AutoReconnect: if mongo == self.__pools.get(self.__writer): self.disconnect() raise errors = [] for host in helpers.shuffled(self.__readers): try: mongo = self.__pools[host] return host, self.__send_and_receive(mongo, msg, **kwargs) except AutoReconnect, why: errors.append(str(why))
def initialize(self): super().initialize() from pymongo.helpers import shuffled self.shuffled_collection = shuffled(image_collection.find()) self.counter = 0