def test_connect(self): txyam.client.reactor = MemoryReactor() YamClient(['one', ('two', 123)]) connection = txyam.client.reactor.connectors[0].getDestination() self.assertEqual(connection, IPv4Address('TCP', 'one', 11211)) connection = txyam.client.reactor.connectors[1].getDestination() self.assertEqual(connection, IPv4Address('TCP', 'two', 123))
def test_stats(self): """ Ensure request goes to all servers. """ addrs = [ IPv4Address('TCP', 'one', 123), IPv4Address('TCP', 'two', 456) ] client = YamClient(addrs, connect=False) transports = makeTestConnections(client) send = "stats\r\n" recv = [ "STAT foo bar\r\nSTAT egg spam\r\nEND\r\n", "STAT baz bar\r\nEND\r\n" ] results = { 'one:123': { "foo": "bar", "egg": "spam" }, 'two:456': { "baz": "bar" } } yield self._test(client.stats(), transports, send, recv, results)
def test_disconnect(self): client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) client.disconnect() for factory in client.factories: self.assertFalse(factory.continueTrying) for transport in transports: self.assertFalse(transport.connected) self.assertTrue(transport.protocol._disconnected)
def test_getClientWithConsistentHashing(self): """ Ensure that the client that has a key remains the same if the total number of hosts goes up or down. """ yclient = YamClient(map(str, range(9)), connect=False) makeTestConnections(yclient) self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf')) yclient = YamClient(map(str, range(10)), connect=False) makeTestConnections(yclient) self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf')) yclient = YamClient(map(str, range(11)), connect=False) makeTestConnections(yclient) self.assertEqual(yclient.factories[6].client, yclient.getClient('sdusdfsdf'))
def test_getClientIsDistributed(self): yclient = YamClient(map(str, range(10)), connect=False) makeTestConnections(yclient) counts = {f.client: 0 for f in yclient.factories} for _ in xrange(200): client = yclient.getClient(str(uuid.uuid4())) counts[client] += 1 # Expect at least 5% of the values are stored on each client for count in counts.values(): self.assertTrue(count > 10)
def test_flushAll(self): """ Ensure that we can flush all servers. """ client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) send = "flush_all\r\n" recv = ["OK\r\n", "OK\r\n"] results = [(True, True), (True, True)] yield self._test(client.flushAll(), transports, send, recv, results)
def test_getPickledWithCompression(self): """ Ensure that pickled object can be fetched with compression. """ client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) # Get a value that should hit first client and not second value = {'foo': 'bar', 'biz': 'baz'} pickled = zlib.compress(cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL)) send = "get aaa\r\n" recv = ["VALUE aaa 0 %i\r\n%s\r\nEND\r\n" % (len(pickled), pickled)] d = client.getPickled("aaa", uncompress=True) yield self._test(d, transports[:1], send, recv, (0, value)) self.assertEqual(transports[1].value(), "")
def test_setPickledWithCompression(self): """ Ensure that pickled object can be stored with compression. """ client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) # Set a value that should hit first client and not second, and is gzipped value = zlib.compress( cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL)) send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value) recv = ["STORED\r\n"] d = client.setPickled("aaa", {'foo': 'bar'}, compress=True) yield self._test(d, transports[:1], send, recv, True) self.assertEqual(transports[1].value(), "")
def test_version(self): """ Ensure request goes to all servers. """ addrs = [ IPv4Address('TCP', 'one', 123), IPv4Address('TCP', 'two', 456) ] client = YamClient(addrs, connect=False) transports = makeTestConnections(client) send = "version\r\n" recv = ["VERSION 1.2\r\n", "VERSION 3.4\r\n"] results = {'one:123': "1.2", 'two:456': "3.4"} yield self._test(client.version(), transports, send, recv, results)
def test_no_servers(self): """ Ensure that exception is thrown if no servers are present. """ client = YamClient([], connect=False) transports = makeTestConnections(client) # flush should be OK with no servers send = "flush_all\r\n" recv = ["OK\r\n", "OK\r\n"] yield self._test(client.flushAll(), transports, send, recv, []) # these, however, shoudl be an issue self.assertRaises(NoServerError, client.set, "foo", "bar") self.assertRaises(NoServerError, client.get, "foo")
def test_getClient(self): """ Ensure that we can split by key correctly. """ yclient = YamClient(['one', 'two'], connect=False) makeTestConnections(yclient) self.assertEqual(yclient.getClient('aaa'), yclient.factories[0].client) self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client) # now lose first connection yclient.factories[0].stopTrying() # next line is handled by clientConnectionLost which is called when an actual # internet.tcp.Connector has a failed connection yclient.factories[0].client = None self.assertEqual(yclient.getClient('aaa'), yclient.factories[1].client) self.assertEqual(yclient.getClient('foo'), yclient.factories[1].client)
def test_lost_connection(self): # now, try with server that has valid transport that # has lost its connection client = YamClient(['one'], connect=False) transports = makeTestConnections(client) client.factories[0].stopTrying() d1 = client.get("foo") d2 = client.get("bar") transports[0].loseConnection() done = DeferredList([d1, d2], consumeErrors=True) def checkFailures(results): for success, result in results: self.assertFalse(success) result.trap(ConnectionDone) return done.addCallback(checkFailures)
def test_sharding_gets(self): """ Ensure that the commands are correctly split by key and sent to the correct client. """ client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) # Get a value that should hit first client and not second send = "get aaa\r\n" recv = ["VALUE aaa 0 3\r\nbar\r\nEND\r\n"] yield self._test(client.get("aaa"), transports[:1], send, recv, (0, "bar")) self.assertEqual(transports[1].value(), "") # Get a value that should hit second client and not first send = "get foo\r\n" recv = ["VALUE foo 0 3\r\nbar\r\nEND\r\n"] transports[0].clear() yield self._test(client.get("foo"), transports[1:], send, recv, (0, "bar")) self.assertEqual(transports[0].value(), "")
def test_setPickled(self): """ Ensure that pickled object can be stored. """ client = YamClient(['one', 'two'], connect=False) transports = makeTestConnections(client) # Set a value that should hit first client and not second value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL) send = "set aaa 0 0 %i\r\n%s\r\n" % (len(value), value) recv = ["STORED\r\n"] yield self._test(client.setPickled("aaa", {'foo': 'bar'}), transports[:1], send, recv, True) self.assertEqual(transports[1].value(), "") # Set a value that should hit second client and not first value = cPickle.dumps({'foo': 'bar'}, cPickle.HIGHEST_PROTOCOL) send = "set foo 0 0 %i\r\n%s\r\n" % (len(value), value) recv = ["STORED\r\n"] transports[0].clear() yield self._test(client.setPickled("foo", {'foo': 'bar'}), transports[1:], send, recv, True) self.assertEqual(transports[0].value(), "")
DBPOOL = common.get_database("afos") MEMCACHE_EXCLUDE = [ "RR1", "RR2", "RR3", "RR4", "RR5", "RR6", "RR7", "RR8", "RR9", "ROB", "HML", ] MEMCACHE_CLIENT = YamClient(reactor, ["tcp:iem-memcached3:11211"]) MEMCACHE_CLIENT.connect() def shutdown(): """ Down we go! """ log.msg("Stopping...") reactor.callWhenRunning(reactor.stop) # @UndefinedVariable # LDM Ingestor class MyProductIngestor(ldmbridge.LDMProductReceiver): """ I receive products from ldmbridge and process them 1 by 1 :) """ def connectionLost(self, reason): """ called when the connection is lost """ log.msg("connectionLost")
def __init__(self, arguments): self.client = SynchronousYamClient( YamClient(arguments.pop('reactor'), arguments.pop('url'), **arguments)) self.client.yamClient.connect()