def test_stats(self):
"""
Ensure request goes to all servers.
"""
addrs = [
IPv4Address('TCP', 'one', 123),
IPv4Address('TCP', 'two', 456)
]
client = YamClient(addrs, connect=False)
transports = makeTestConnections(client)
send = "stats\r\n"
recv = [
"STAT foo bar\r\nSTAT egg spam\r\nEND\r\n",
"STAT baz bar\r\nEND\r\n"
]
results = {
'one:123': {
"foo": "bar",
"egg": "spam"
},
'two:456': {
"baz": "bar"
}
}
yield self._test(client.stats(), transports, send, recv, results)
def test_disconnect(self):
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
client.disconnect()
for factory in client.factories:
self.assertFalse(factory.continueTrying)
for transport in transports:
self.assertFalse(transport.connected)
self.assertTrue(transport.protocol._disconnected)
def test_disconnect(self):
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
client.disconnect()
for factory in client.factories:
self.assertFalse(factory.continueTrying)
for transport in transports:
self.assertFalse(transport.connected)
self.assertTrue(transport.protocol._disconnected)
def __init__(self, hosts, queue, *args, **kwargs):
"""
@param hosts: List of hosts, will be passed directly to
YamClient.__init__.
@param queue: Anything that conforms to DeferredQueue's interface.
"""
self.queue = queue
self.connectingDeferred = None
YamClient.__init__(self, hosts, *args, **kwargs)
def test_getClientIsDistributed(self):
yclient = YamClient(map(str, range(10)), connect=False)
makeTestConnections(yclient)
counts = {f.client: 0 for f in yclient.factories}
for _ in xrange(200):
client = yclient.getClient(str(uuid.uuid4()))
counts[client] += 1
# Expect at least 5% of the values are stored on each client
for count in counts.values():
self.assertTrue(count > 10)
def test_flushAll(self):
"""
Ensure that we can flush all servers.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
send = "flush_all\r\n"
recv = ["OK\r\n", "OK\r\n"]
results = [(True, True), (True, True)]
yield self._test(client.flushAll(), transports, send, recv, results)
def test_flushAll(self):
"""
Ensure that we can flush all servers.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
send = "flush_all\r\n"
recv = ["OK\r\n", "OK\r\n"]
results = [(True, True), (True, True)]
yield self._test(client.flushAll(), transports, send, recv, results)
def test_getClientIsDistributed(self):
yclient = YamClient(map(str, range(10)), connect=False)
makeTestConnections(yclient)
counts = {f.client: 0 for f in yclient.factories}
for _ in xrange(200):
client = yclient.getClient(str(uuid.uuid4()))
counts[client] += 1
# Expect at least 5% of the values are stored on each client
for count in counts.values():
self.assertTrue(count > 10)
def test_version(self):
"""
Ensure request goes to all servers.
"""
addrs = [IPv4Address('TCP', 'one', 123), IPv4Address('TCP', 'two', 456)]
client = YamClient(addrs, connect=False)
transports = makeTestConnections(client)
send = "version\r\n"
recv = ["VERSION 1.2\r\n", "VERSION 3.4\r\n"]
results = {'one:123': "1.2", 'two:456': "3.4"}
yield self._test(client.version(), transports, send, recv, results)
def test_stats(self):
"""
Ensure request goes to all servers.
"""
addrs = [IPv4Address('TCP', 'one', 123), IPv4Address('TCP', 'two', 456)]
client = YamClient(addrs, connect=False)
transports = makeTestConnections(client)
send = "stats\r\n"
recv = ["STAT foo bar\r\nSTAT egg spam\r\nEND\r\n", "STAT baz bar\r\nEND\r\n"]
results = {'one:123': {"foo": "bar", "egg": "spam"}, 'two:456': {"baz": "bar"}}
yield self._test(client.stats(), transports, send, recv, results)
def test_getPickled(self):
"""
Ensure that pickled object can be fetched.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Get a value that should hit first client and not second
value = {'foo': 'bar', 'biz': 'baz'}
pickled = cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL)
send = "get aaa\r\n"
recv = ["VALUE aaa 0 %i\r\n%s\r\nEND\r\n" % (len(pickled), pickled)]
yield self._test(client.getPickled("aaa"), transports[:1], send, recv, (0, value))
self.assertEqual(transports[1].value(), "")
def test_setPickledWithCompression(self):
"""
Ensure that pickled object can be stored with compression.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Set a value that should hit first client and not second, and is gzipped
value = zlib.compress(cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL))
send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
d = client.setPickled("aaa", {'foo': 'bar'}, compress=True)
yield self._test(d, transports[:1], send, recv, True)
self.assertEqual(transports[1].value(), "")
def test_no_servers(self):
"""
Ensure that exception is thrown if no servers are present.
"""
client = YamClient([], connect=False)
transports = makeTestConnections(client)
# flush should be OK with no servers
send = "flush_all\r\n"
recv = ["OK\r\n", "OK\r\n"]
yield self._test(client.flushAll(), transports, send, recv, [])
# these, however, shoudl be an issue
self.assertRaises(NoServerError, client.set, "foo", "bar")
self.assertRaises(NoServerError, client.get, "foo")
def test_no_servers(self):
"""
Ensure that exception is thrown if no servers are present.
"""
client = YamClient([], connect=False)
transports = makeTestConnections(client)
# flush should be OK with no servers
send = "flush_all\r\n"
recv = ["OK\r\n", "OK\r\n"]
yield self._test(client.flushAll(), transports, send, recv, [])
# these, however, shoudl be an issue
self.assertRaises(NoServerError, client.set, "foo", "bar")
self.assertRaises(NoServerError, client.get, "foo")
def test_setPickledWithCompression(self):
"""
Ensure that pickled object can be stored with compression.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Set a value that should hit first client and not second, and is gzipped
value = zlib.compress(
cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL))
send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
d = client.setPickled("aaa", {'foo': 'bar'}, compress=True)
yield self._test(d, transports[:1], send, recv, True)
self.assertEqual(transports[1].value(), "")
def test_getPickledWithCompression(self):
"""
Ensure that pickled object can be fetched with compression.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Get a value that should hit first client and not second
value = {'foo': 'bar', 'biz': 'baz'}
pickled = zlib.compress(cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL))
send = "get aaa\r\n"
recv = ["VALUE aaa 0 %i\r\n%s\r\nEND\r\n" % (len(pickled), pickled)]
d = client.getPickled("aaa", uncompress=True)
yield self._test(d, transports[:1], send, recv, (0, value))
self.assertEqual(transports[1].value(), "")
def test_version(self):
"""
Ensure request goes to all servers.
"""
addrs = [
IPv4Address('TCP', 'one', 123),
IPv4Address('TCP', 'two', 456)
]
client = YamClient(addrs, connect=False)
transports = makeTestConnections(client)
send = "version\r\n"
recv = ["VERSION 1.2\r\n", "VERSION 3.4\r\n"]
results = {'one:123': "1.2", 'two:456': "3.4"}
yield self._test(client.version(), transports, send, recv, results)
def test_getClient(self):
"""
Ensure that we can split by key correctly.
"""
yclient = YamClient(['one', 'two'], connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.getClient('aaa'), yclient.factories[0].client)
self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client)
# now lose first connection
yclient.factories[0].stopTrying()
# next line is handled by clientConnectionLost which is called when an actual
# internet.tcp.Connector has a failed connection
yclient.factories[0].client = None
self.assertEqual(yclient.getClient('aaa'), yclient.factories[1].client)
self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client)
def test_connect(self):
txyam.client.reactor = MemoryReactor()
YamClient(['one', ('two', 123)])
connection = txyam.client.reactor.connectors[0].getDestination()
self.assertEqual(connection, IPv4Address('TCP', 'one', 11211))
connection = txyam.client.reactor.connectors[1].getDestination()
self.assertEqual(connection, IPv4Address('TCP', 'two', 123))
def test_getClient(self):
"""
Ensure that we can split by key correctly.
"""
yclient = YamClient(['one', 'two'], connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.getClient('aaa'), yclient.factories[0].client)
self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client)
# now lose first connection
yclient.factories[0].stopTrying()
# next line is handled by clientConnectionLost which is called when an actual
# internet.tcp.Connector has a failed connection
yclient.factories[0].client = None
self.assertEqual(yclient.getClient('aaa'), yclient.factories[1].client)
self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client)
def test_lost_connection(self):
# now, try with server that has valid transport that
# has lost its connection
client = YamClient(['one'], connect=False)
transports = makeTestConnections(client)
client.factories[0].stopTrying()
d1 = client.get("foo")
d2 = client.get("bar")
transports[0].loseConnection()
done = DeferredList([d1, d2], consumeErrors=True)
def checkFailures(results):
for success, result in results:
self.assertFalse(success)
result.trap(ConnectionDone)
return done.addCallback(checkFailures)
def test_lost_connection(self):
# now, try with server that has valid transport that
# has lost its connection
client = YamClient(['one'], connect=False)
transports = makeTestConnections(client)
client.factories[0].stopTrying()
d1 = client.get("foo")
d2 = client.get("bar")
transports[0].loseConnection()
done = DeferredList([d1, d2], consumeErrors=True)
def checkFailures(results):
for success, result in results:
self.assertFalse(success)
result.trap(ConnectionDone)
return done.addCallback(checkFailures)
def test_sharding_gets(self):
"""
Ensure that the commands are correctly split by key and sent
to the correct client.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Get a value that should hit first client and not second
send = "get aaa\r\n"
recv = ["VALUE aaa 0 3\r\nbar\r\nEND\r\n"]
yield self._test(client.get("aaa"), transports[:1], send, recv, (0, "bar"))
self.assertEqual(transports[1].value(), "")
# Get a value that should hit second client and not first
send = "get foo\r\n"
recv = ["VALUE foo 0 3\r\nbar\r\nEND\r\n"]
transports[0].clear()
yield self._test(client.get("foo"), transports[1:], send, recv, (0, "bar"))
self.assertEqual(transports[0].value(), "")
def test_setPickled(self):
"""
Ensure that pickled object can be stored.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Set a value that should hit first client and not second
value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL)
send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
yield self._test(client.setPickled("aaa", {'foo': 'bar'}), transports[:1], send, recv, True)
self.assertEqual(transports[1].value(), "")
# Set a value that should hit second client and not first
value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL)
send = "set foo 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
transports[0].clear()
yield self._test(client.setPickled("foo", {'foo': 'bar'}), transports[1:], send, recv, True)
self.assertEqual(transports[0].value(), "")
def test_sharding_gets(self):
"""
Ensure that the commands are correctly split by key and sent
to the correct client.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Get a value that should hit first client and not second
send = "get aaa\r\n"
recv = ["VALUE aaa 0 3\r\nbar\r\nEND\r\n"]
yield self._test(client.get("aaa"), transports[:1], send, recv,
(0, "bar"))
self.assertEqual(transports[1].value(), "")
# Get a value that should hit second client and not first
send = "get foo\r\n"
recv = ["VALUE foo 0 3\r\nbar\r\nEND\r\n"]
transports[0].clear()
yield self._test(client.get("foo"), transports[1:], send, recv,
(0, "bar"))
self.assertEqual(transports[0].value(), "")
def test_getClientWithConsistentHashing(self):
"""
Ensure that the client that has a key remains the same if the total number
of hosts goes up or down.
"""
yclient = YamClient(map(str, range(9)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client,
yclient.getClient('sdusdfsdf'))
yclient = YamClient(map(str, range(10)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client,
yclient.getClient('sdusdfsdf'))
yclient = YamClient(map(str, range(11)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client,
yclient.getClient('sdusdfsdf'))
def test_setPickled(self):
"""
Ensure that pickled object can be stored.
"""
client = YamClient(['one', 'two'], connect=False)
transports = makeTestConnections(client)
# Set a value that should hit first client and not second
value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL)
send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
yield self._test(client.setPickled("aaa", {'foo': 'bar'}),
transports[:1], send, recv, True)
self.assertEqual(transports[1].value(), "")
# Set a value that should hit second client and not first
value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL)
send = "set foo 0 0 %i\r\n%s\r\n" % (len(value), value)
recv = ["STORED\r\n"]
transports[0].clear()
yield self._test(client.setPickled("foo", {'foo': 'bar'}),
transports[1:], send, recv, True)
self.assertEqual(transports[0].value(), "")
def test_getClientWithConsistentHashing(self):
"""
Ensure that the client that has a key remains the same if the total number
of hosts goes up or down.
"""
yclient = YamClient(map(str, range(9)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf'))
yclient = YamClient(map(str, range(10)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf'))
yclient = YamClient(map(str, range(11)), connect=False)
makeTestConnections(yclient)
self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf'))
DBPOOL = common.get_database("afos")
MEMCACHE_EXCLUDE = [
"RR1",
"RR2",
"RR3",
"RR4",
"RR5",
"RR6",
"RR7",
"RR8",
"RR9",
"ROB",
"HML",
]
MEMCACHE_CLIENT = YamClient(reactor, ["tcp:iem-memcached3:11211"])
MEMCACHE_CLIENT.connect()
def shutdown():
""" Down we go! """
log.msg("Stopping...")
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" called when the connection is lost """
log.msg("connectionLost")
import sys
import datetime
import pytz
from twisted.python import log
from twisted.internet import reactor
from txyam.client import YamClient
from pyldm import ldmbridge
from pyiem.nws import product
from pyiem.util import utc
import common # @UnresolvedImport
DBPOOL = common.get_database('afos')
MEMCACHE_EXCLUDE = ['RR1', 'RR2', 'RR3', 'RR4', 'RR5', 'RR6', 'RR7', 'RR8',
'RR9', 'ROB', 'HML']
MEMCACHE_CLIENT = YamClient(reactor, ['tcp:iem-memcached3:11211', ])
MEMCACHE_CLIENT.connect()
def shutdown():
""" Down we go! """
log.msg("Stopping...")
reactor.callWhenRunning(reactor.stop) # @UndefinedVariable
# LDM Ingestor
class MyProductIngestor(ldmbridge.LDMProductReceiver):
""" I receive products from ldmbridge and process them 1 by 1 :) """
def connectionLost(self, reason):
""" called when the connection is lost """
def poolDisconnect(self):
"""poolDisconnect: Actually disconnect this client. This should only be
called by the controlling pool.
"""
return YamClient.disconnect(self)
def connect(self):
if self.connectingDeferred is None:
self.connectingDeferred = YamClient.connect(self)
return self.connectingDeferred # XXX: What happens when the factory
def __init__(self, arguments):
self.client = SynchronousYamClient(
YamClient(arguments.pop('reactor'), arguments.pop('url'),
**arguments))
self.client.yamClient.connect()
