def _send_message_with_response(self, msg, _connection_to_use=None, _must_use_master=False, **kwargs):
"""Send a message to Mongo and return the response.
Sends the given message and returns the response.
:Parameters:
- `msg`: (request_id, data) pair making up the message to send
"""
read_pref = kwargs.get("read_preference", ReadPreference.PRIMARY)
mongo = None
try:
if _connection_to_use is not None:
if _connection_to_use == -1:
mongo = self.__find_primary()
else:
mongo = self.__pools[_connection_to_use]
return mongo["pool"].host, self.__send_and_receive(mongo, msg, **kwargs)
elif _must_use_master or not read_pref:
mongo = self.__find_primary()
return mongo["pool"].host, self.__send_and_receive(mongo, msg, **kwargs)
except AutoReconnect:
if mongo == self.__writer:
self.disconnect()
raise
errors = []
for host in helpers.shuffled(self.__readers):
try:
mongo = self.__pools[host]
return host, self.__send_and_receive(mongo, msg, **kwargs)
except AutoReconnect, why:
errors.append(str(why))
def _send_message_with_response(self, message, _connection_to_use=None, _must_use_master=False, **kwargs):
"""Receive a message from Mongo.
Sends the given message and returns a (connection_id, response) pair.
:Parameters:
- `operation`: opcode of the message to send
- `data`: data to send
"""
if _connection_to_use is not None:
if _connection_to_use == -1:
return (-1, self.__master._send_message_with_response(message, **kwargs))
else:
return (
_connection_to_use,
self.__slaves[_connection_to_use]._send_message_with_response(message, **kwargs),
)
# _must_use_master is set for commands, which must be sent to the
# master instance. any queries in a request must be sent to the
# master since that is where writes go.
if _must_use_master or self.__in_request:
return (-1, self.__master._send_message_with_response(message, **kwargs))
# Iterate through the slaves randomly until we have success. Raise
# reconnect if they all fail.
for connection_id in helpers.shuffled(range(len(self.__slaves))):
try:
slave = self.__slaves[connection_id]
return (connection_id, slave._send_message_with_response(message, **kwargs))
except AutoReconnect:
pass
raise AutoReconnect("failed to connect to slaves")
def kooaba_upload(count):
from kooaba.bridge import kooaba_bridge
from pymongo.helpers import shuffled
logger = logging.getLogger(__name__)
logger.info("Kooaba uploading " + str(count) + " images")
images = shuffled(image_collection.find())
for i in range(count):
image = images[i]
logger.info("Start uploading " + str(image))
kooaba_bridge.upload_image(image)
def _send_message_with_response(self,
message,
_connection_to_use=None,
_must_use_master=False,
**kwargs):
"""Receive a message from Mongo.
Sends the given message and returns a (connection_id, response) pair.
:Parameters:
- `operation`: opcode of the message to send
- `data`: data to send
"""
if _connection_to_use is not None:
if _connection_to_use == -1:
member = self.__master
conn = -1
else:
member = self.__slaves[_connection_to_use]
conn = _connection_to_use
return (conn,
member._send_message_with_response(message, **kwargs)[1])
# _must_use_master is set for commands, which must be sent to the
# master instance. any queries in a request must be sent to the
# master since that is where writes go.
if _must_use_master or self.in_request():
return (-1,
self.__master._send_message_with_response(
message, **kwargs)[1])
# Iterate through the slaves randomly until we have success. Raise
# reconnect if they all fail.
for connection_id in helpers.shuffled(xrange(len(self.__slaves))):
try:
slave = self.__slaves[connection_id]
return (connection_id,
slave._send_message_with_response(message,
**kwargs)[1])
except AutoReconnect:
pass
raise AutoReconnect("failed to connect to slaves")
def _send_message_with_response(self,
msg,
_connection_to_use=None,
_must_use_master=False,
**kwargs):
"""Send a message to Mongo and return the response.
Sends the given message and returns the response.
:Parameters:
- `msg`: (request_id, data) pair making up the message to send
"""
read_pref = kwargs.get('read_preference', ReadPreference.PRIMARY)
mongo = None
try:
if _connection_to_use is not None:
if _connection_to_use == -1:
mongo = self.__find_primary()
else:
mongo = self.__pools[_connection_to_use]
return mongo['pool'].pair, self.__send_and_receive(
mongo, msg, **kwargs)
elif _must_use_master or not read_pref:
mongo = self.__find_primary()
return mongo['pool'].pair, self.__send_and_receive(
mongo, msg, **kwargs)
except AutoReconnect:
if mongo == self.__pools.get(self.__writer):
self.disconnect()
raise
errors = []
for host in helpers.shuffled(self.__readers):
try:
mongo = self.__pools[host]
return host, self.__send_and_receive(mongo, msg, **kwargs)
except AutoReconnect, why:
errors.append(str(why))
def initialize(self):
super().initialize()
from pymongo.helpers import shuffled
self.shuffled_collection = shuffled(image_collection.find())
self.counter = 0
