def append_prepend_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.append(b'key', b'after', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'valueafter') result = client.prepend(b'key1', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key1') tools.assert_equal(result, None) result = client.prepend(b'key', b'before', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'beforevalueafter')
def append_prepend_test(host, port): client = Client((host, port)) client.flush_all() result = client.append('key', 'value', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.append('key', 'after', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'valueafter') result = client.prepend('key1', 'value', noreply=False) tools.assert_equal(result, False) result = client.get('key1') tools.assert_equal(result, None) result = client.prepend('key', 'before', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port): client = Client((host, port)) client.flush_all() result = client.append('key', 'value', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.append('key', 'after', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'valueafter') result = client.prepend('key1', 'value', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key1') tools.assert_equal(result, None) result = client.prepend('key', 'before', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.append(b'key', b'after', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'valueafter') result = client.prepend(b'key1', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key1') tools.assert_equal(result, None) result = client.prepend(b'key', b'before', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'beforevalueafter')
def test_append_prepend(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) assert result is False result = client.get(b'key') assert result is None result = client.set(b'key', b'value', noreply=False) assert result is True result = client.append(b'key', b'after', noreply=False) assert result is True result = client.get(b'key') assert result == b'valueafter' result = client.prepend(b'key1', b'value', noreply=False) assert result is False result = client.get(b'key1') assert result is None result = client.prepend(b'key', b'before', noreply=False) assert result is True result = client.get(b'key') assert result == b'beforevalueafter'
def delete_test(host, port): client = Client((host, port)) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.delete('key', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, None)
def delete_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.delete(b'key', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.delete(b'key', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, None)
def delete_test(host, port): client = Client((host, port)) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.delete('key', noreply=False) tools.assert_equal(result, 'DELETED') result = client.get('key') tools.assert_equal(result, None)
def test_delete(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.delete(b'key', noreply=False) assert result is False result = client.get(b'key') assert result is None result = client.set(b'key', b'value', noreply=False) assert result is True result = client.delete(b'key', noreply=False) assert result is True result = client.get(b'key') assert result is None
def delete_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.delete(b'key', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.delete(b'key', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, None)
def delete_test(host, port): client = Client((host, port)) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.delete('key', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, None)
def application(environ, start_response): """Do Fun things""" form = parse_formvars(environ) cb = form.get("callback", None) huc12 = form.get("huc12", "000000000000")[:12] mode = form.get("mode", "daily") fmt = form.get("format", "json") if fmt == "json": headers = [("Content-Type", "application/vnd.geo+json")] elif fmt == "xlsx": headers = [ ("Content-Type", EXL), ("Content-disposition", f"attachment; Filename=dep{huc12}.xlsx"), ] start_response("200 OK", headers) mckey = f"/geojson/huc12_events/{huc12}/{mode}/{fmt}" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(huc12, mode, fmt) if fmt == "xlsx": mc.close() return [res] mc.set(mckey, res, 15) else: res = res.decode("utf-8") mc.close() if cb is not None: res = f"{cb}({res})" return [res.encode("ascii")]
def test_incr_decr(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) assert result is None result = client.set(b'key', b'0', noreply=False) assert result is True result = client.incr(b'key', 1, noreply=False) assert result == 1 def _bad_int(): client.incr(b'key', b'foobar') with pytest.raises(MemcacheClientError): _bad_int() result = client.decr(b'key1', 1, noreply=False) assert result is None result = client.decr(b'key', 1, noreply=False) assert result == 0 result = client.get(b'key') assert result == b'0'
def application(environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) form = parse_formvars(environ) cb = form.get("callback", None) domain = form.get("domain", None) ts = datetime.datetime.strptime(form.get("date", "2015-05-05"), "%Y-%m-%d") ts2 = None if form.get("date2", None) is not None: ts2 = datetime.datetime.strptime(form.get("date2"), "%Y-%m-%d") tkey = "" if ts2 is None else ts2.strftime("%Y%m%d") dkey = "" if domain is None else domain mckey = f"/geojson/huc12/{ts:%Y%m%d}/{tkey}/{dkey}" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(ts, ts2, domain) try: # Unknown BrokenPipeError mc.set(mckey, res, 3600) except Exception as exp: print(exp) else: res = res.decode("utf-8") mc.close() if cb is not None: res = f"{cb}({res})" return [res.encode("ascii")]
def get_set_test(host, port): client = Client((host, port)) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set('key', 'value', noreply=False) result = client.get('key') tools.assert_equal(result, 'value') client.set('key2', 'value2', noreply=True) result = client.get('key2') tools.assert_equal(result, 'value2') result = client.get_many(['key', 'key2']) tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) def get(self, query, params): return self.memcached_client.get(key_serializer(query,params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query,params),document)
def get_set_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set(b'key', b'value', noreply=False) result = client.get(b'key') tools.assert_equal(result, b'value') client.set(b'key2', b'value2', noreply=True) result = client.get(b'key2') tools.assert_equal(result, b'value2') result = client.get_many([b'key', b'key2']) tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'}) result = client.get_many([]) tools.assert_equal(result, {})
def test_get_set(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') assert result is None client.set(b'key', b'value', noreply=False) result = client.get(b'key') assert result == b'value' client.set(b'key2', b'value2', noreply=True) result = client.get(b'key2') assert result == b'value2' result = client.get_many([b'key', b'key2']) assert result == {b'key': b'value', b'key2': b'value2'} result = client.get_many([]) assert result == {}
def get_set_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set(b'key', b'value', noreply=False) result = client.get(b'key') tools.assert_equal(result, b'value') client.set(b'key2', b'value2', noreply=True) result = client.get(b'key2') tools.assert_equal(result, b'value2') result = client.get_many([b'key', b'key2']) tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'}) result = client.get_many([]) tools.assert_equal(result, {})
def get_set_test(host, port): client = Client((host, port)) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set('key', 'value', noreply=False) result = client.get('key') tools.assert_equal(result, 'value') client.set('key2', 'value2', noreply=True) result = client.get('key2') tools.assert_equal(result, 'value2') result = client.get_many(['key', 'key2']) tools.assert_equal(result, {'key': 'value', 'key2': 'value2'}) result = client.get_many([]) tools.assert_equal(result, {})
def get_or_set(key, generator): if is_active: client = Client((host, port)) key = __build_key(key) result = client.get(key) if not result: result = generator() client.set(key, result) client.quit() return result else: return generator()
def get_or_set(key,generator): if is_active: client = Client((host,port)) key = __build_key(key) result = client.get(key) if not result: result = generator() client.set(key, result) client.quit() return result else: return generator()
class Dao: def __init__(self, host, port): self.db = Client( (host, port), default_noreply=False, serializer=self.__json_serializer, deserializer=self.__json_deserializer ) def __json_serializer(self, key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 def __json_deserializer(self, key, value, flags): if flags == 1: return value.decode('utf-8') if flags == 2: return json.loads(value.decode('utf-8')) raise Exception('Unknown serialization format') def get_count(self): count = self.db.get('count') return int(count) if count else 0 def get_sushi_list(self, count): ids = range(1, count+1) result = self.db.get_multi([f'sushi:{id}' for id in ids]) return [{'id': id, **result[f'sushi:{id}']} for id in ids] def incr_count(self): count = self.db.get('count') if count: return self.db.incr('count', 1) else: self.db.set('count', 1) return 1 def set_sushi(self, sushi): self.db.set('sushi:'+str(sushi['id']), sushi)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): try: from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) except ImportError: raise DependencyError("Error importing pymemcache library for OnionMemcached") def get(self, query, params): return self.memcached_client.get(key_serializer(query,params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query,params),document)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) def get(self, query, params): return self.memcached_client.get(key_serializer(query, params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query, params), document)
def add_replace_test(host, port): client = Client((host, port)) client.flush_all() result = client.add('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'value') result = client.add('key', 'value2', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, 'value') result = client.replace('key1', 'value1', noreply=False) tools.assert_equal(result, False) result = client.get('key1') tools.assert_equal(result, None) result = client.replace('key', 'value2', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'value2')
def add_replace_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.add(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'value') result = client.add(b'key', b'value2', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, b'value') result = client.replace(b'key1', b'value1', noreply=False) tools.assert_equal(result, False) result = client.get(b'key1') tools.assert_equal(result, None) result = client.replace(b'key', b'value2', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'value2')
def test_add_replace(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.add(b'key', b'value', noreply=False) assert result is True result = client.get(b'key') assert result == b'value' result = client.add(b'key', b'value2', noreply=False) assert result is False result = client.get(b'key') assert result == b'value' result = client.replace(b'key1', b'value1', noreply=False) assert result is False result = client.get(b'key1') assert result is None result = client.replace(b'key', b'value2', noreply=False) assert result is True result = client.get(b'key') assert result == b'value2'
def add_replace_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.add(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'value') result = client.add(b'key', b'value2', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, b'value') result = client.replace(b'key1', b'value1', noreply=False) tools.assert_equal(result, False) result = client.get(b'key1') tools.assert_equal(result, None) result = client.replace(b'key', b'value2', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'value2')
def add_replace_test(host, port): client = Client((host, port)) client.flush_all() result = client.add('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'value') result = client.add('key', 'value2', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key') tools.assert_equal(result, 'value') result = client.replace('key1', 'value1', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key1') tools.assert_equal(result, None) result = client.replace('key', 'value2', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'value2')
def application(_environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) mckey = "/geojson/huc12.geojson" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do() mc.set(mckey, res, 86400) else: res = res.decode("utf-8") mc.close() return [res.encode("ascii")]
class Cache: def __init__(self): self.__mem = Client(**CACHE_CONFIGS) # memcache init. here self._tasks = 0 def __len__(self): return self._tasks def set(self, task, user_id): try: if self.__mem.set(str(user_id), task.json()): self._tasks += 1 except (MemcacheIllegalInputError, MemcacheUnknownCommandError, TypeError) as e: logging.critical("%s occured with setting(%s: %s)" % (str(e), str(user_id), str(task.json()))) def get(self, user_id): try: res = self.__mem.get(str(user_id)) except (MemcacheIllegalInputError, MemcacheUnknownCommandError, TypeError) as e: logging.critical("%s occured with getting %s" % (str(e), str(user_id))) return None return res def pop(self, user_id): try: res = self.__mem.get(str(user_id)) self.__mem.delete(str(user_id)) except (MemcacheIllegalInputError, MemcacheUnknownCommandError, TypeError) as e: logging.critical("%s occured with popping %s" % (str(e), str(user_id))) return None return res
def on_get(self, req, resp): # Go to position frame = get_frame() avg_color = get_average_color_of_frame(file) threat_level = rgb_to_threat(avg_color) memcached_client = Client(('localhost', 11211)) state = json.loads(memcached_client.get('state')) pos = (state['x'], state['y']) resp.body = json.dumps({ 'position': pos, 'threat_value': threat_level, }) resp.status = falcon.HTTP_200
def test_serialization_deserialization(host, port): def _ser(key, value): return json.dumps(value), 1 def _des(key, value, flags): if flags == 1: return json.loads(value) return value client = Client((host, port), serializer=_ser, deserializer=_des) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set('key', value) result = client.get('key') tools.assert_equal(result, value)
def test_serialization_deserialization(host, port): def _ser(value): return json.dumps(value), 1 def _des(value, flags): if flags == 1: return json.loads(value) return value client = Client((host, port), serializer=_ser, deserializer=_des) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set('key', value) result = client.get('key') tools.assert_equal(result, value)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): try: from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) except ImportError: raise DependencyError( "Error importing pymemcache library for OnionMemcached") def get(self, query, params): return self.memcached_client.get(key_serializer(query, params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query, params), document)
class PyMemcachedCache(BaseCache): """A cache client based on pymemcache. implemented by pure python and support noreply. """ def __init__(self, config): BaseCache.__init__(self, config) self._client = PyMemcachedClient((config['host'], config['port']), serializer=python_memcache_serializer, deserializer=python_memcache_deserializer, connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT, key_prefix=config.get('key_prefix', '')) def get(self, key): return self._client.get(key) def delete(self, key, noreply=False): self._client.delete(key, noreply) return True def get_many(self, keys): return self._client.get_many(keys) def set(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set(key, value, timeout, noreply) def add(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.add(key, value, timeout, noreply) def set_many(self, data, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set_many(data, timeout, noreply) def delete_many(self, keys, noreply=False): return self._client.delete_many(keys, noreply) def clear(self): return self._client.flush_all(noreply=False) def incr(self, key, delta=1, noreply=False): return self._client.incr(key, delta, noreply) def decr(self, key, delta=1, noreply=False): return self._client.decr(key, delta, noreply)
class PyMemcachedCache(BaseCache): """A cache client based on pymemcache. implemented by pure python and support noreply. """ def __init__(self, config): BaseCache.__init__(self, config) self._client = PyMemcachedClient((config['host'], config['port']), serializer=python_memcache_serializer, deserializer=python_memcache_deserializer, connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT, key_prefix=config.get('key_prefix', '')) def get(self, key): return self._client.get(key) def delete(self, key, noreply=False): self._client.delete(key, noreply) return True def get_many(self, keys): return self._client.get_many(keys) def set(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set(key, value, timeout, noreply) def add(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.add(key, value, timeout, noreply) def set_many(self, data, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set_many(data, timeout, noreply) def delete_many(self, keys, noreply=False): return self._client.delete_many(keys, noreply) def clear(self): return self._client.flush_all(noreply=False) def incr(self, key, delta=1, noreply=False): return self._client.incr(key, delta, noreply) def decr(self, key, delta=1, noreply=False): return self._client.decr(key, delta, noreply)
def test_serialization_deserialization(host, port, socket_module): def _ser(key, value): return json.dumps(value).encode('ascii'), 1 def _des(key, value, flags): if flags == 1: return json.loads(value.decode('ascii')) return value client = Client((host, port), serializer=_ser, deserializer=_des, socket_module=socket_module) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set(b'key', value) result = client.get(b'key') assert result == value
def application(environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) form = parse_formvars(environ) cb = form.get("callback", None) huc12 = form.get("huc12", "000000000000")[:12] mode = form.get("mode", "daily") mckey = "/geojson/huc12_events/%s/%s" % (huc12, mode) mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(huc12, mode) mc.set(mckey, res, 15) else: res = res.decode("utf-8") mc.close() if cb is not None: res = "%s(%s)" % (cb, res) return [res.encode("ascii")]
def main(environ): """Do something fun""" form = parse_formvars(environ) year = form.get("year", 2015) month = form.get("month", 5) day = form.get("day", 5) year2 = form.get("year2", year) month2 = form.get("month2", month) day2 = form.get("day2", day) scenario = int(form.get("scenario", 0)) v = form.get("v", "avg_loss") huc = form.get("huc") ts = datetime.date(int(year), int(month), int(day)) ts2 = datetime.date(int(year2), int(month2), int(day2)) mckey = "/auto/map.py/%s/%s/%s/%s/%s" % ( huc, ts.strftime("%Y%m%d"), ts2.strftime("%Y%m%d"), scenario, v, ) if form.get("overview"): mckey = "/auto/map.py/%s/%s" % (huc, form.get("zoom")) mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: # Lazy import to help speed things up if form.get("overview"): res, do_cache = make_overviewmap(form) else: res, do_cache = make_map(huc, ts, ts2, scenario, v, form) sys.stderr.write("Setting cache: %s\n" % (mckey,)) if do_cache: mc.set(mckey, res, 3600) mc.close() return res
def incr_decr_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, None) result = client.set(b'key', b'0', noreply=False) tools.assert_equal(result, True) result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, 1) def _bad_int(): client.incr(b'key', b'foobar') tools.assert_raises(MemcacheClientError, _bad_int) result = client.decr(b'key1', 1, noreply=False) tools.assert_equal(result, None) result = client.decr(b'key', 1, noreply=False) tools.assert_equal(result, 0) result = client.get(b'key') tools.assert_equal(result, b'0')
def incr_decr_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, None) result = client.set(b'key', b'0', noreply=False) tools.assert_equal(result, True) result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, 1) def _bad_int(): client.incr(b'key', b'foobar') tools.assert_raises(MemcacheClientError, _bad_int) result = client.decr(b'key1', 1, noreply=False) tools.assert_equal(result, None) result = client.decr(b'key', 1, noreply=False) tools.assert_equal(result, 0) result = client.get(b'key') tools.assert_equal(result, b'0')
def incr_decr_test(host, port): client = Client((host, port)) client.flush_all() result = client.incr('key', 1, noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.set('key', '0', noreply=False) tools.assert_equal(result, 'STORED') result = client.incr('key', 1, noreply=False) tools.assert_equal(result, 1) def _bad_int(): client.incr('key', 'foobar') tools.assert_raises(MemcacheClientError, _bad_int) result = client.decr('key1', 1, noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.decr('key', 1, noreply=False) tools.assert_equal(result, 0) result = client.get('key') tools.assert_equal(result, '0')
import requests import re from pymemcache.client import Client from xml.dom.minidom import parseString import sys # User defined variables feedaddr = 'http://malwaredb.malekal.com/export.php?type=url' feedID = 'malekal_malware' mkey = 'fetcher_malekal_malware:feeddata' killchain = 'Malware' # No user modifications needed below. client = Client(('localhost', 11211)) result = client.get(mkey) if isinstance(result, str): dom = parseString(result) else: r = requests.get(feedaddr) client.set(mkey, r.content, expire=7200) dom = parseString(r.content) xmlroot = dom.getElementsByTagName("uris")[0] print('url,feedID,killchain,description,method') for entry in xmlroot.getElementsByTagName("url"): url = entry.getElementsByTagName('address')[0].firstChild if url is None: continue url = url.nodeValue.strip() msplit = url.split('://')
import requests import re from pymemcache.client import Client # User defined variables feedaddr = 'https://feodotracker.abuse.ch/blocklist.php?download=ipblocklist' feedID = 'feodotracker' mkey = 'fetcher_feodoIP:feeddata' killchain = 'Command & Control' # No user modifications needed below. client = Client(('localhost', 11211)) result = client.get(mkey) if isinstance(result, str): splitlines = result.split('\n') else: r = requests.get(feedaddr) client.set(mkey, r.content, expire=7200) splitlines = r.content.split('\n') print('ipv4,feedID,killchain,description') for x in splitlines: x = x.strip() if re.search('^#', x): continue if len(x) == 0: continue print("%s,%s,%s,%s" % (x, feedID, killchain, 'Feodo C&C'))
logger.addHandler(fh) ## Open memcache mc = MemcacheClient(('127.0.0.1', 11211), serializer=json_serializer, deserializer=json_deserializer) ## NS Notifier userkey (will come from url/cli parameter in the future) try: userkey = settings.userkey except AttributeError: userkey = 1 ## Are we planned to run? (E.g., not disabled through web) try: should_run = mc.get('nsapi_run') except: should_run = True if should_run == None: should_run = True #logger.info('no run tuple in memcache, creating') mc.set('nsapi_run', should_run) #print('should run? ' + str(should_run)) logger.debug('Should run: ' + str(should_run)) if not should_run: sys.exit(0) errors = [] nsapi = ns_api.NSAPI(settings.username, settings.apikey)
class MemcachedStore(object): """Caching implementation that uses Memcached as data storage. :param host: String representation of hostname or IP of the memcached server :param port: Port number (int) on which the memcached server is listening :param connect_timeout: optional float, seconds to wait for a connection to the memcached server. Defaults to "forever" (uses the underlying default socket timeout, which can be very long) :param timeout: optional float, seconds to wait for send or recv calls on the socket connected to memcached. Defaults to "forever" (uses the underlying default socket timeout, which can be very long). :param no_delay: optional bool, set the TCP_NODELAY flag, which may help with performance in some cases. Defaults to False. :param ignore_exc: optional bool, True to cause the "get", "gets", "get_many" and "gets_many" calls to treat any errors as cache misses. Defaults to True. Ie. if the cache is failing use the Stormpath API. :param socket_module: socket module to use, e.g. gevent.socket. Defaults to the standard library's socket module. :param key_prefix: Prefix of key. You can use this as namespace. Defaults to b''. """ DEFAULT_TTL = 5 * 60 # seconds def __init__(self, host='localhost', port=11211, connect_timeout=None, timeout=None, no_delay=False, ignore_exc=True, key_prefix=b'', socket_module=socket, ttl=DEFAULT_TTL): self.ttl = ttl try: from pymemcache.client import Client as Memcache except ImportError: raise RuntimeError('Memcached support is not available. Run "pip install pymemcache".') self.memcache = Memcache( (host, port), serializer=json_serializer, deserializer=json_deserializer, connect_timeout=connect_timeout, timeout=timeout, socket_module=socket_module, no_delay=no_delay, ignore_exc=ignore_exc, key_prefix=key_prefix) @memcache_error_handling def __getitem__(self, key): entry = self.memcache.get(key) if entry is None: return None return CacheEntry.parse(entry) @memcache_error_handling def __setitem__(self, key, entry): self.memcache.set(key, entry, expire=self.ttl) @memcache_error_handling def __delitem__(self, key): self.memcache.delete(key) @memcache_error_handling def clear(self): self.memcache.flush_all() @memcache_error_handling def __len__(self): return self.memcache.stats()['curr_items']
class MemcachedStore(object): """Caching implementation that uses Memcached as data storage. :param host: String representation of hostname or IP of the memcached server :param port: Port number (int) on which the memcached server is listening :param connect_timeout: optional float, seconds to wait for a connection to the memcached server. Defaults to "forever" (uses the underlying default socket timeout, which can be very long) :param timeout: optional float, seconds to wait for send or recv calls on the socket connected to memcached. Defaults to "forever" (uses the underlying default socket timeout, which can be very long). :param no_delay: optional bool, set the TCP_NODELAY flag, which may help with performance in some cases. Defaults to False. :param ignore_exc: optional bool, True to cause the "get", "gets", "get_many" and "gets_many" calls to treat any errors as cache misses. Defaults to True. Ie. if the cache is failing use the Stormpath API. :param socket_module: socket module to use, e.g. gevent.socket. Defaults to the standard library's socket module. :param key_prefix: Prefix of key. You can use this as namespace. Defaults to b''. """ DEFAULT_TTL = 5 * 60 # seconds def __init__(self, host='localhost', port=11211, connect_timeout=None, timeout=None, no_delay=False, ignore_exc=True, key_prefix=b'', socket_module=socket, ttl=DEFAULT_TTL): self.ttl = ttl try: from pymemcache.client import Client as Memcache except ImportError: raise RuntimeError( 'Memcached support is not available. Run "pip install pymemcache".' ) self.memcache = Memcache((host, port), serializer=json_serializer, deserializer=json_deserializer, connect_timeout=connect_timeout, timeout=timeout, socket_module=socket_module, no_delay=no_delay, ignore_exc=ignore_exc, key_prefix=key_prefix) @memcache_error_handling def __getitem__(self, key): entry = self.memcache.get(key) if entry is None: return None return CacheEntry.parse(entry) @memcache_error_handling def __setitem__(self, key, entry): self.memcache.set(key, entry, expire=self.ttl) @memcache_error_handling def __delitem__(self, key): self.memcache.delete(key) @memcache_error_handling def clear(self): self.memcache.flush_all() @memcache_error_handling def __len__(self): return self.memcache.stats()['curr_items']
def test_get_not_found(): client = Client(None) client.sock = MockSocket(['END\r\n']) result = client.get('key') tools.assert_equal(result, None)
def test_get_found(): client = Client(None) client.sock = MockSocket(['VALUE key 0 5\r\nvalue\r\nEND\r\n']) result = client.get('key') tools.assert_equal(result, 'value')
def test_get_recv_chunks(): client = Client(None) client.sock = MockSocket(['VALUE key', ' 0 5\r', '\nvalue', '\r\n', 'END', '\r', '\n']) result = client.get('key') tools.assert_equal(result, 'value')
from pymemcache.client import Client c = Client(("localhost", 11211)) from time import time c.flush_all(noreply=False) start = time() num = 1000 for x in range(num): c.set(str(x), str(x), noreply=False) val = c.get(str(x)) assert val == str(x), str(x) print "incrementing monkey (expecting a fail)" result = c.incr("monkey", 1, noreply=False) assert result == None print "setting monkey" print c.set("monkey", 0, noreply=False) zero = c.get("monkey") assert zero == "0", zero print "incrementing monkey" print c.incr("monkey", 1, noreply=False) monkey = c.get("monkey") assert monkey == "1", monkey print c.decr("monkey", 1, noreply=False)