def test_socket_connect(self):
server = ("example.com", 11211)
client = Client(server, socket_module=MockSocketModule())
client._connect()
tools.assert_equal(client.sock.connections, [server])
timeout = 2
connect_timeout = 3
client = Client(server,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=MockSocketModule())
client._connect()
tools.assert_equal(client.sock.timeouts, [connect_timeout, timeout])
client = Client(server, socket_module=MockSocketModule())
client._connect()
tools.assert_equal(client.sock.socket_options, [])
client = Client(server,
socket_module=MockSocketModule(),
no_delay=True)
client._connect()
tools.assert_equal(client.sock.socket_options,
[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)])
def test_socket_connect(self):
server = ("example.com", 11211)
client = Client(server, socket_module=MockSocketModule())
client._connect()
assert client.sock.connections == [server]
timeout = 2
connect_timeout = 3
client = Client(server,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=MockSocketModule())
client._connect()
assert client.sock.timeouts == [connect_timeout, timeout]
client = Client(server, socket_module=MockSocketModule())
client._connect()
assert client.sock.socket_options == []
client = Client(server,
socket_module=MockSocketModule(),
no_delay=True)
client._connect()
assert client.sock.socket_options == [(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)]
def application(environ, start_response):
"""Do Fun things"""
headers = [("Content-Type", "application/vnd.geo+json")]
start_response("200 OK", headers)
form = parse_formvars(environ)
cb = form.get("callback", None)
domain = form.get("domain", None)
ts = datetime.datetime.strptime(form.get("date", "2015-05-05"), "%Y-%m-%d")
ts2 = None
if form.get("date2", None) is not None:
ts2 = datetime.datetime.strptime(form.get("date2"), "%Y-%m-%d")
tkey = "" if ts2 is None else ts2.strftime("%Y%m%d")
dkey = "" if domain is None else domain
mckey = f"/geojson/huc12/{ts:%Y%m%d}/{tkey}/{dkey}"
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do(ts, ts2, domain)
try:
# Unknown BrokenPipeError
mc.set(mckey, res, 3600)
except Exception as exp:
print(exp)
else:
res = res.decode("utf-8")
mc.close()
if cb is not None:
res = f"{cb}({res})"
return [res.encode("ascii")]
def __init__(self, host, port):
self.db = Client(
(host, port),
default_noreply=False,
serializer=self.__json_serializer,
deserializer=self.__json_deserializer
)
def test_append_prepend(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.append(b'key', b'after', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'valueafter'
result = client.prepend(b'key1', b'value', noreply=False)
assert result is False
result = client.get(b'key1')
assert result is None
result = client.prepend(b'key', b'before', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'beforevalueafter'
def get_value(user_input=None, client=Client((HOST, PORT)), db_name=DATABASE):
"""gets the value from key from memcached or sqlite"""
if user_input is None:
return None
response = client.get(user_input[1])
if response is None:
LOG.info('%s not found in memcached, searching sqlite', user_input[1])
db_conn = KeyValueHelper(db_name)
try:
response = db_conn[user_input[1]]
LOG.info('%s found in sqlite. Adding to memcahed', user_input[1])
client.set(user_input[1], response)
except KeyError as error:
LOG.info('%s not found in sqlite', user_input[1])
LOG.error(error)
db_conn.close()
else:
response = response.decode('utf-8')
return response
def test_quit():
client = Client(None)
client.sock = MockSocket([])
result = client.quit()
tools.assert_equal(result, None)
tools.assert_equal(client.sock, None)
tools.assert_equal(client.buf, '')
def test_incr_decr(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
assert result is None
result = client.set(b'key', b'0', noreply=False)
assert result is True
result = client.incr(b'key', 1, noreply=False)
assert result == 1
def _bad_int():
client.incr(b'key', b'foobar')
with pytest.raises(MemcacheClientError):
_bad_int()
result = client.decr(b'key1', 1, noreply=False)
assert result is None
result = client.decr(b'key', 1, noreply=False)
assert result == 0
result = client.get(b'key')
assert result == b'0'
def test_errors(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
def _key_with_ws():
client.set('key with spaces', 'value', noreply=False)
tools.assert_raises(MemcacheIllegalInputError, _key_with_ws)
def _key_too_long():
client.set('x' * 1024, 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _key_too_long)
def _unicode_key_in_set():
client.set(u'\u0FFF', 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_key_in_set)
def _unicode_key_in_get():
client.get(u'\u0FFF')
tools.assert_raises(MemcacheClientError, _unicode_key_in_get)
def _unicode_value_in_set():
client.set('key', u'\u0FFF', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_value_in_set)
def append_prepend_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.append('key', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.append('key', 'after', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'valueafter')
result = client.prepend('key1', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key1')
tools.assert_equal(result, None)
result = client.prepend('key', 'before', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'beforevalueafter')
def test_errors(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
def _key_with_ws():
client.set(b'key with spaces', b'value', noreply=False)
with pytest.raises(MemcacheIllegalInputError):
_key_with_ws()
def _key_too_long():
client.set(b'x' * 1024, b'value', noreply=False)
with pytest.raises(MemcacheClientError):
_key_too_long()
def _unicode_key_in_set():
client.set(six.u('\u0FFF'), b'value', noreply=False)
with pytest.raises(MemcacheClientError):
_unicode_key_in_set()
def _unicode_key_in_get():
client.get(six.u('\u0FFF'))
with pytest.raises(MemcacheClientError):
_unicode_key_in_get()
def _unicode_value_in_set():
client.set(b'key', six.u('\u0FFF'), noreply=False)
with pytest.raises(MemcacheClientError):
_unicode_value_in_set()
def test_get_many_all_found():
client = Client(None)
client.sock = MockSocket(
['VALUE key1 0 6\r\nvalue1\r\n'
'VALUE key2 0 6\r\nvalue2\r\nEND\r\n'])
result = client.get_many(['key1', 'key2'])
tools.assert_equal(result, {'key1': 'value1', 'key2': 'value2'})
def application(environ, start_response):
"""Do Fun things"""
form = parse_formvars(environ)
cb = form.get("callback", None)
huc12 = form.get("huc12", "000000000000")[:12]
mode = form.get("mode", "daily")
fmt = form.get("format", "json")
if fmt == "json":
headers = [("Content-Type", "application/vnd.geo+json")]
elif fmt == "xlsx":
headers = [
("Content-Type", EXL),
("Content-disposition", f"attachment; Filename=dep{huc12}.xlsx"),
]
start_response("200 OK", headers)
mckey = f"/geojson/huc12_events/{huc12}/{mode}/{fmt}"
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do(huc12, mode, fmt)
if fmt == "xlsx":
mc.close()
return [res]
mc.set(mckey, res, 15)
else:
res = res.decode("utf-8")
mc.close()
if cb is not None:
res = f"{cb}({res})"
return [res.encode("ascii")]
def test_memcache_storage(self):
from pymemcache.client import Client
from tornadoapi.storage.kvstorage import KvStorage
servers = ("127.0.0.1", 11211)
memcached = Client(servers)
storage = KvStorage(memcached)
self.test_caches(storage)
def append_prepend_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.append(b'key', b'after', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'valueafter')
result = client.prepend(b'key1', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.prepend(b'key', b'before', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'beforevalueafter')
def test_stats_conversions():
client = Client(None)
client.sock = MockSocket([
# Most stats are converted to int
'STAT cmd_get 2519\r\n',
'STAT cmd_set 3099\r\n',
# Unless they can't be, they remain str
'STAT libevent 2.0.19-stable\r\n',
# Some named stats are explicitly converted
'STAT hash_is_expanding 0\r\n',
'STAT rusage_user 0.609165\r\n',
'STAT rusage_system 0.852791\r\n',
'STAT slab_reassign_running 1\r\n',
'STAT version 1.4.14\r\n',
'END\r\n',
])
result = client.stats()
tools.assert_equal(client.sock.send_bufs, ['stats \r\n'])
expected = {
'cmd_get': 2519,
'cmd_set': 3099,
'libevent': '2.0.19-stable',
'hash_is_expanding': False,
'rusage_user': 0.609165,
'rusage_system': 0.852791,
'slab_reassign_running': True,
'version': '1.4.14',
}
tools.assert_equal(result, expected)
def test_set_success():
client = Client(None)
client.sock = MockSocket(['STORED\r\n'])
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
tools.assert_equal(client.sock.closed, False)
tools.assert_equal(len(client.sock.send_bufs), 1)
def test_get_unknown_error():
client = Client(None)
client.sock = MockSocket(['foobarbaz\r\n'])
def _get():
client.get('key')
tools.assert_raises(MemcacheUnknownError, _get)
def test_get_error():
client = Client(None)
client.sock = MockSocket(['ERROR\r\n'])
def _get():
client.get('key')
tools.assert_raises(MemcacheUnknownCommandError, _get)
def test_set_unknown_error():
client = Client(None)
client.sock = MockSocket(['foobarbaz\r\n'])
def _set():
client.set('key', 'value', noreply=False)
tools.assert_raises(MemcacheUnknownError, _set)
def test_get_unicode_key():
client = Client(None)
client.sock = MockSocket([''])
def _get():
client.get(u'\u0FFF')
tools.assert_raises(MemcacheIllegalInputError, _get)
def test_set_server_error():
client = Client(None)
client.sock = MockSocket(['SERVER_ERROR some message\r\n'])
def _set():
client.set('key', 'value', noreply=False)
tools.assert_raises(MemcacheServerError, _set)
def test_set_unicode_value():
client = Client(None)
client.sock = MockSocket([''])
def _set():
client.set('key', u'\u0FFF', noreply=False)
tools.assert_raises(MemcacheIllegalInputError, _set)
def test_set_error():
client = Client(None)
client.sock = MockSocket(['ERROR\r\n'])
def _set():
client.set('key', 'value', noreply=False)
tools.assert_raises(MemcacheUnknownCommandError, _set)
def __init__(self, host=('localhost', 11211)):
try:
from pymemcache.client import Client
self.memcached_client = Client(host,
serializer=json_serializer,
deserializer=json_deserializer)
except ImportError:
raise DependencyError(
"Error importing pymemcache library for OnionMemcached")
def __init__(self, ip, port):
super(Memcached, self).__init__()
# link = [str(ip) + ":" + str(port)]
self.mc = Client(
(ip, int(port)),
serializer=serialize_json,
deserializer=deserialize_json,
connect_timeout=2,
)
def test_serialization():
def _ser(key, value):
return json.dumps(value), 0
client = Client(None, serializer=_ser)
client.sock = MockSocket(['STORED\r\n'])
client.set('key', {'a': 'b', 'c': 'd'})
tools.assert_equal(client.sock.send_bufs,
['set key 0 0 20 noreply\r\n{"a": "b", "c": "d"}\r\n'])
def test_delete_exception():
client = Client(None)
client.sock = MockSocket([Exception('fail')])
def _delete():
client.delete('key', noreply=False)
tools.assert_raises(Exception, _delete)
tools.assert_equal(client.sock, None)
tools.assert_equal(client.buf, '')
def test_incr_exception():
client = Client(None)
client.sock = MockSocket([Exception('fail')])
def _incr():
client.incr('key', 1)
tools.assert_raises(Exception, _incr)
tools.assert_equal(client.sock, None)
tools.assert_equal(client.buf, '')
def __init__(self,
host='192.168.24.138',
port=11211,
timeout=5,
ignore_exc=True):
self.rhost = host
self.rport = port
self.rcli = Client((host, port),
timeout=timeout,
ignore_exc=ignore_exc)
