def append_prepend_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.append(b'key', b'after', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'valueafter')
result = client.prepend(b'key1', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.prepend(b'key', b'before', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'beforevalueafter')
def append_prepend_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.append('key', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.append('key', 'after', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'valueafter')
result = client.prepend('key1', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key1')
tools.assert_equal(result, None)
result = client.prepend('key', 'before', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.append('key', 'value', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.append('key', 'after', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'valueafter')
result = client.prepend('key1', 'value', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key1')
tools.assert_equal(result, None)
result = client.prepend('key', 'before', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.append(b'key', b'after', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'valueafter')
result = client.prepend(b'key1', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.prepend(b'key', b'before', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'beforevalueafter')
def test_append_prepend(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.append(b'key', b'after', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'valueafter'
result = client.prepend(b'key1', b'value', noreply=False)
assert result is False
result = client.get(b'key1')
assert result is None
result = client.prepend(b'key', b'before', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'beforevalueafter'
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
def delete_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'DELETED')
result = client.get('key')
tools.assert_equal(result, None)
def test_delete(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.delete(b'key', noreply=False)
assert result is True
result = client.get(b'key')
assert result is None
def delete_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
def application(environ, start_response):
"""Do Fun things"""
form = parse_formvars(environ)
cb = form.get("callback", None)
huc12 = form.get("huc12", "000000000000")[:12]
mode = form.get("mode", "daily")
fmt = form.get("format", "json")
if fmt == "json":
headers = [("Content-Type", "application/vnd.geo+json")]
elif fmt == "xlsx":
headers = [
("Content-Type", EXL),
("Content-disposition", f"attachment; Filename=dep{huc12}.xlsx"),
]
start_response("200 OK", headers)
mckey = f"/geojson/huc12_events/{huc12}/{mode}/{fmt}"
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do(huc12, mode, fmt)
if fmt == "xlsx":
mc.close()
return [res]
mc.set(mckey, res, 15)
else:
res = res.decode("utf-8")
mc.close()
if cb is not None:
res = f"{cb}({res})"
return [res.encode("ascii")]
def test_incr_decr(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
assert result is None
result = client.set(b'key', b'0', noreply=False)
assert result is True
result = client.incr(b'key', 1, noreply=False)
assert result == 1
def _bad_int():
client.incr(b'key', b'foobar')
with pytest.raises(MemcacheClientError):
_bad_int()
result = client.decr(b'key1', 1, noreply=False)
assert result is None
result = client.decr(b'key', 1, noreply=False)
assert result == 0
result = client.get(b'key')
assert result == b'0'
def application(environ, start_response):
"""Do Fun things"""
headers = [("Content-Type", "application/vnd.geo+json")]
start_response("200 OK", headers)
form = parse_formvars(environ)
cb = form.get("callback", None)
domain = form.get("domain", None)
ts = datetime.datetime.strptime(form.get("date", "2015-05-05"), "%Y-%m-%d")
ts2 = None
if form.get("date2", None) is not None:
ts2 = datetime.datetime.strptime(form.get("date2"), "%Y-%m-%d")
tkey = "" if ts2 is None else ts2.strftime("%Y%m%d")
dkey = "" if domain is None else domain
mckey = f"/geojson/huc12/{ts:%Y%m%d}/{tkey}/{dkey}"
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do(ts, ts2, domain)
try:
# Unknown BrokenPipeError
mc.set(mckey, res, 3600)
except Exception as exp:
print(exp)
else:
res = res.decode("utf-8")
mc.close()
if cb is not None:
res = f"{cb}({res})"
return [res.encode("ascii")]
def get_set_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set('key', 'value', noreply=False)
result = client.get('key')
tools.assert_equal(result, 'value')
client.set('key2', 'value2', noreply=True)
result = client.get('key2')
tools.assert_equal(result, 'value2')
result = client.get_many(['key', 'key2'])
tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
class OnionMemcached(OnionCache):
def __init__(self, host=('localhost', 11211)):
from pymemcache.client import Client
self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer)
def get(self, query, params):
return self.memcached_client.get(key_serializer(query,params))
def set(self, query, params, document):
return self.memcached_client.set(key_serializer(query,params),document)
def get_set_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
tools.assert_equal(result, b'value2')
result = client.get_many([b'key', b'key2'])
tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def test_get_set(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
assert result is None
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
assert result == b'value'
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
assert result == b'value2'
result = client.get_many([b'key', b'key2'])
assert result == {b'key': b'value', b'key2': b'value2'}
result = client.get_many([])
assert result == {}
def get_set_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
tools.assert_equal(result, b'value2')
result = client.get_many([b'key', b'key2'])
tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def get_set_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set('key', 'value', noreply=False)
result = client.get('key')
tools.assert_equal(result, 'value')
client.set('key2', 'value2', noreply=True)
result = client.get('key2')
tools.assert_equal(result, 'value2')
result = client.get_many(['key', 'key2'])
tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def get_or_set(key, generator):
if is_active:
client = Client((host, port))
key = __build_key(key)
result = client.get(key)
if not result:
result = generator()
client.set(key, result)
client.quit()
return result
else:
return generator()
def get_or_set(key,generator):
if is_active:
client = Client((host,port))
key = __build_key(key)
result = client.get(key)
if not result:
result = generator()
client.set(key, result)
client.quit()
return result
else:
return generator()
class Dao:
def __init__(self, host, port):
self.db = Client(
(host, port),
default_noreply=False,
serializer=self.__json_serializer,
deserializer=self.__json_deserializer
)
def __json_serializer(self, key, value):
if type(value) == str:
return value, 1
return json.dumps(value), 2
def __json_deserializer(self, key, value, flags):
if flags == 1:
return value.decode('utf-8')
if flags == 2:
return json.loads(value.decode('utf-8'))
raise Exception('Unknown serialization format')
def get_count(self):
count = self.db.get('count')
return int(count) if count else 0
def get_sushi_list(self, count):
ids = range(1, count+1)
result = self.db.get_multi([f'sushi:{id}' for id in ids])
return [{'id': id, **result[f'sushi:{id}']} for id in ids]
def incr_count(self):
count = self.db.get('count')
if count:
return self.db.incr('count', 1)
else:
self.db.set('count', 1)
return 1
def set_sushi(self, sushi):
self.db.set('sushi:'+str(sushi['id']), sushi)
class OnionMemcached(OnionCache):
def __init__(self, host=('localhost', 11211)):
try:
from pymemcache.client import Client
self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer)
except ImportError:
raise DependencyError("Error importing pymemcache library for OnionMemcached")
def get(self, query, params):
return self.memcached_client.get(key_serializer(query,params))
def set(self, query, params, document):
return self.memcached_client.set(key_serializer(query,params),document)
class OnionMemcached(OnionCache):
def __init__(self, host=('localhost', 11211)):
from pymemcache.client import Client
self.memcached_client = Client(host,
serializer=json_serializer,
deserializer=json_deserializer)
def get(self, query, params):
return self.memcached_client.get(key_serializer(query, params))
def set(self, query, params, document):
return self.memcached_client.set(key_serializer(query, params),
document)
def add_replace_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.add('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.add('key', 'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.replace('key1', 'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get('key1')
tools.assert_equal(result, None)
result = client.replace('key', 'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'value2')
def add_replace_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.add(b'key', b'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.replace(b'key1', b'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.replace(b'key', b'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value2')
def test_add_replace(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'value'
result = client.add(b'key', b'value2', noreply=False)
assert result is False
result = client.get(b'key')
assert result == b'value'
result = client.replace(b'key1', b'value1', noreply=False)
assert result is False
result = client.get(b'key1')
assert result is None
result = client.replace(b'key', b'value2', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'value2'
def add_replace_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.add(b'key', b'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.replace(b'key1', b'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.replace(b'key', b'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value2')
def add_replace_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.add('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.add('key', 'value2', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.replace('key1', 'value1', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key1')
tools.assert_equal(result, None)
result = client.replace('key', 'value2', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'value2')
def application(_environ, start_response):
"""Do Fun things"""
headers = [("Content-Type", "application/vnd.geo+json")]
start_response("200 OK", headers)
mckey = "/geojson/huc12.geojson"
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do()
mc.set(mckey, res, 86400)
else:
res = res.decode("utf-8")
mc.close()
return [res.encode("ascii")]
class Cache:
def __init__(self):
self.__mem = Client(**CACHE_CONFIGS) # memcache init. here
self._tasks = 0
def __len__(self):
return self._tasks
def set(self, task, user_id):
try:
if self.__mem.set(str(user_id), task.json()):
self._tasks += 1
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with setting(%s: %s)" %
(str(e), str(user_id), str(task.json())))
def get(self, user_id):
try:
res = self.__mem.get(str(user_id))
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with getting %s" %
(str(e), str(user_id)))
return None
return res
def pop(self, user_id):
try:
res = self.__mem.get(str(user_id))
self.__mem.delete(str(user_id))
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with popping %s" %
(str(e), str(user_id)))
return None
return res
def on_get(self, req, resp):
# Go to position
frame = get_frame()
avg_color = get_average_color_of_frame(file)
threat_level = rgb_to_threat(avg_color)
memcached_client = Client(('localhost', 11211))
state = json.loads(memcached_client.get('state'))
pos = (state['x'], state['y'])
resp.body = json.dumps({
'position': pos,
'threat_value': threat_level,
})
resp.status = falcon.HTTP_200
def test_serialization_deserialization(host, port):
def _ser(key, value):
return json.dumps(value), 1
def _des(key, value, flags):
if flags == 1:
return json.loads(value)
return value
client = Client((host, port), serializer=_ser, deserializer=_des)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set('key', value)
result = client.get('key')
tools.assert_equal(result, value)
def test_serialization_deserialization(host, port):
def _ser(value):
return json.dumps(value), 1
def _des(value, flags):
if flags == 1:
return json.loads(value)
return value
client = Client((host, port), serializer=_ser, deserializer=_des)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set('key', value)
result = client.get('key')
tools.assert_equal(result, value)
class OnionMemcached(OnionCache):
def __init__(self, host=('localhost', 11211)):
try:
from pymemcache.client import Client
self.memcached_client = Client(host,
serializer=json_serializer,
deserializer=json_deserializer)
except ImportError:
raise DependencyError(
"Error importing pymemcache library for OnionMemcached")
def get(self, query, params):
return self.memcached_client.get(key_serializer(query, params))
def set(self, query, params, document):
return self.memcached_client.set(key_serializer(query, params),
document)
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
def test_serialization_deserialization(host, port, socket_module):
def _ser(key, value):
return json.dumps(value).encode('ascii'), 1
def _des(key, value, flags):
if flags == 1:
return json.loads(value.decode('ascii'))
return value
client = Client((host, port), serializer=_ser, deserializer=_des,
socket_module=socket_module)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set(b'key', value)
result = client.get(b'key')
assert result == value
def application(environ, start_response):
"""Do Fun things"""
headers = [("Content-Type", "application/vnd.geo+json")]
start_response("200 OK", headers)
form = parse_formvars(environ)
cb = form.get("callback", None)
huc12 = form.get("huc12", "000000000000")[:12]
mode = form.get("mode", "daily")
mckey = "/geojson/huc12_events/%s/%s" % (huc12, mode)
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
res = do(huc12, mode)
mc.set(mckey, res, 15)
else:
res = res.decode("utf-8")
mc.close()
if cb is not None:
res = "%s(%s)" % (cb, res)
return [res.encode("ascii")]
def main(environ):
"""Do something fun"""
form = parse_formvars(environ)
year = form.get("year", 2015)
month = form.get("month", 5)
day = form.get("day", 5)
year2 = form.get("year2", year)
month2 = form.get("month2", month)
day2 = form.get("day2", day)
scenario = int(form.get("scenario", 0))
v = form.get("v", "avg_loss")
huc = form.get("huc")
ts = datetime.date(int(year), int(month), int(day))
ts2 = datetime.date(int(year2), int(month2), int(day2))
mckey = "/auto/map.py/%s/%s/%s/%s/%s" % (
huc,
ts.strftime("%Y%m%d"),
ts2.strftime("%Y%m%d"),
scenario,
v,
)
if form.get("overview"):
mckey = "/auto/map.py/%s/%s" % (huc, form.get("zoom"))
mc = Client(["iem-memcached", 11211])
res = mc.get(mckey)
if res is None:
# Lazy import to help speed things up
if form.get("overview"):
res, do_cache = make_overviewmap(form)
else:
res, do_cache = make_map(huc, ts, ts2, scenario, v, form)
sys.stderr.write("Setting cache: %s\n" % (mckey,))
if do_cache:
mc.set(mckey, res, 3600)
mc.close()
return res
def incr_decr_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'0', noreply=False)
tools.assert_equal(result, True)
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr(b'key', b'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr(b'key1', 1, noreply=False)
tools.assert_equal(result, None)
result = client.decr(b'key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get(b'key')
tools.assert_equal(result, b'0')
def incr_decr_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'0', noreply=False)
tools.assert_equal(result, True)
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr(b'key', b'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr(b'key1', 1, noreply=False)
tools.assert_equal(result, None)
result = client.decr(b'key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get(b'key')
tools.assert_equal(result, b'0')
def incr_decr_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.incr('key', 1, noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.set('key', '0', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.incr('key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr('key', 'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr('key1', 1, noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.decr('key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get('key')
tools.assert_equal(result, '0')
import requests
import re
from pymemcache.client import Client
from xml.dom.minidom import parseString
import sys
# User defined variables
feedaddr = 'http://malwaredb.malekal.com/export.php?type=url'
feedID = 'malekal_malware'
mkey = 'fetcher_malekal_malware:feeddata'
killchain = 'Malware'
# No user modifications needed below.
client = Client(('localhost', 11211))
result = client.get(mkey)
if isinstance(result, str):
dom = parseString(result)
else:
r = requests.get(feedaddr)
client.set(mkey, r.content, expire=7200)
dom = parseString(r.content)
xmlroot = dom.getElementsByTagName("uris")[0]
print('url,feedID,killchain,description,method')
for entry in xmlroot.getElementsByTagName("url"):
url = entry.getElementsByTagName('address')[0].firstChild
if url is None: continue
url = url.nodeValue.strip()
msplit = url.split('://')
import requests
import re
from pymemcache.client import Client
# User defined variables
feedaddr = 'https://feodotracker.abuse.ch/blocklist.php?download=ipblocklist'
feedID = 'feodotracker'
mkey = 'fetcher_feodoIP:feeddata'
killchain = 'Command & Control'
# No user modifications needed below.
client = Client(('localhost', 11211))
result = client.get(mkey)
if isinstance(result, str):
splitlines = result.split('\n')
else:
r = requests.get(feedaddr)
client.set(mkey, r.content, expire=7200)
splitlines = r.content.split('\n')
print('ipv4,feedID,killchain,description')
for x in splitlines:
x = x.strip()
if re.search('^#', x): continue
if len(x) == 0: continue
print("%s,%s,%s,%s" % (x, feedID, killchain, 'Feodo C&C'))
logger.addHandler(fh)
## Open memcache
mc = MemcacheClient(('127.0.0.1', 11211), serializer=json_serializer,
deserializer=json_deserializer)
## NS Notifier userkey (will come from url/cli parameter in the future)
try:
userkey = settings.userkey
except AttributeError:
userkey = 1
## Are we planned to run? (E.g., not disabled through web)
try:
should_run = mc.get('nsapi_run')
except:
should_run = True
if should_run == None:
should_run = True
#logger.info('no run tuple in memcache, creating')
mc.set('nsapi_run', should_run)
#print('should run? ' + str(should_run))
logger.debug('Should run: ' + str(should_run))
if not should_run:
sys.exit(0)
errors = []
nsapi = ns_api.NSAPI(settings.username, settings.apikey)
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self, host='localhost', port=11211,
connect_timeout=None, timeout=None,
no_delay=False, ignore_exc=True,
key_prefix=b'', socket_module=socket, ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError('Memcached support is not available. Run "pip install pymemcache".')
self.memcache = Memcache(
(host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self,
host='localhost',
port=11211,
connect_timeout=None,
timeout=None,
no_delay=False,
ignore_exc=True,
key_prefix=b'',
socket_module=socket,
ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError(
'Memcached support is not available. Run "pip install pymemcache".'
)
self.memcache = Memcache((host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
def test_get_not_found():
client = Client(None)
client.sock = MockSocket(['END\r\n'])
result = client.get('key')
tools.assert_equal(result, None)
def test_get_found():
client = Client(None)
client.sock = MockSocket(['VALUE key 0 5\r\nvalue\r\nEND\r\n'])
result = client.get('key')
tools.assert_equal(result, 'value')
def test_get_recv_chunks():
client = Client(None)
client.sock = MockSocket(['VALUE key', ' 0 5\r', '\nvalue', '\r\n',
'END', '\r', '\n'])
result = client.get('key')
tools.assert_equal(result, 'value')
from pymemcache.client import Client
c = Client(("localhost", 11211))
from time import time
c.flush_all(noreply=False)
start = time()
num = 1000
for x in range(num):
c.set(str(x), str(x), noreply=False)
val = c.get(str(x))
assert val == str(x), str(x)
print "incrementing monkey (expecting a fail)"
result = c.incr("monkey", 1, noreply=False)
assert result == None
print "setting monkey"
print c.set("monkey", 0, noreply=False)
zero = c.get("monkey")
assert zero == "0", zero
print "incrementing monkey"
print c.incr("monkey", 1, noreply=False)
monkey = c.get("monkey")
assert monkey == "1", monkey
print c.decr("monkey", 1, noreply=False)
