def application(environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) form = parse_formvars(environ) cb = form.get("callback", None) domain = form.get("domain", None) ts = datetime.datetime.strptime(form.get("date", "2015-05-05"), "%Y-%m-%d") ts2 = None if form.get("date2", None) is not None: ts2 = datetime.datetime.strptime(form.get("date2"), "%Y-%m-%d") tkey = "" if ts2 is None else ts2.strftime("%Y%m%d") dkey = "" if domain is None else domain mckey = f"/geojson/huc12/{ts:%Y%m%d}/{tkey}/{dkey}" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(ts, ts2, domain) try: # Unknown BrokenPipeError mc.set(mckey, res, 3600) except Exception as exp: print(exp) else: res = res.decode("utf-8") mc.close() if cb is not None: res = f"{cb}({res})" return [res.encode("ascii")]
def application(environ, start_response): """Do Fun things""" form = parse_formvars(environ) cb = form.get("callback", None) huc12 = form.get("huc12", "000000000000")[:12] mode = form.get("mode", "daily") fmt = form.get("format", "json") if fmt == "json": headers = [("Content-Type", "application/vnd.geo+json")] elif fmt == "xlsx": headers = [ ("Content-Type", EXL), ("Content-disposition", f"attachment; Filename=dep{huc12}.xlsx"), ] start_response("200 OK", headers) mckey = f"/geojson/huc12_events/{huc12}/{mode}/{fmt}" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(huc12, mode, fmt) if fmt == "xlsx": mc.close() return [res] mc.set(mckey, res, 15) else: res = res.decode("utf-8") mc.close() if cb is not None: res = f"{cb}({res})" return [res.encode("ascii")]
def run_test(): c = Client(("localhost", 11211)) start = time() for x in range(lim): c.set(str(uuid1()), "1") return time()-start
def test_serialization(): def _ser(key, value): return json.dumps(value), 0 client = Client(None, serializer=_ser) client.sock = MockSocket(['STORED\r\n']) client.set('key', {'a': 'b', 'c': 'd'}) tools.assert_equal(client.sock.send_bufs, ['set key 0 0 20 noreply\r\n{"a": "b", "c": "d"}\r\n'])
def test_serialization(): def _ser(value): return json.dumps(value), 0 client = Client(None, serializer=_ser) client.sock = MockSocket(['STORED\r\n']) client.set('key', {'a': 'b', 'c': 'd'}) tools.assert_equal(client.sock.send_bufs, [ 'set key 0 0 20 noreply\r\n{"a": "b", "c": "d"}\r\n' ])
def get_or_set(key,generator): if is_active: client = Client((host,port)) key = __build_key(key) result = client.get(key) if not result: result = generator() client.set(key, result) client.quit() return result else: return generator()
def get_or_set(key, generator): if is_active: client = Client((host, port)) key = __build_key(key) result = client.get(key) if not result: result = generator() client.set(key, result) client.quit() return result else: return generator()
def application(_environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) mckey = "/geojson/huc12.geojson" mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do() mc.set(mckey, res, 86400) else: res = res.decode("utf-8") mc.close() return [res.encode("ascii")]
def test_set_success(): client = Client(None) client.sock = MockSocket(['STORED\r\n']) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') tools.assert_equal(client.sock.closed, False) tools.assert_equal(len(client.sock.send_bufs), 1)
def test_set_success(): client = Client(None) client.sock = MockSocket(['STORED\r\n']) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) tools.assert_equal(client.sock.closed, False) tools.assert_equal(len(client.sock.send_bufs), 1)
def test_incr_decr(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) assert result is None result = client.set(b'key', b'0', noreply=False) assert result is True result = client.incr(b'key', 1, noreply=False) assert result == 1 def _bad_int(): client.incr(b'key', b'foobar') with pytest.raises(MemcacheClientError): _bad_int() result = client.decr(b'key1', 1, noreply=False) assert result is None result = client.decr(b'key', 1, noreply=False) assert result == 0 result = client.get(b'key') assert result == b'0'
def append_prepend_test(host, port): client = Client((host, port)) client.flush_all() result = client.append('key', 'value', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.append('key', 'after', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'valueafter') result = client.prepend('key1', 'value', noreply=False) tools.assert_equal(result, 'NOT_STORED') result = client.get('key1') tools.assert_equal(result, None) result = client.prepend('key', 'before', noreply=False) tools.assert_equal(result, 'STORED') result = client.get('key') tools.assert_equal(result, 'beforevalueafter')
def test_serialization_deserialization(host, port): def _ser(value): return json.dumps(value), 1 def _des(value, flags): if flags == 1: return json.loads(value) return value client = Client((host, port), serializer=_ser, deserializer=_des) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set('key', value) result = client.get('key') tools.assert_equal(result, value)
def test_serialization_deserialization(host, port): def _ser(key, value): return json.dumps(value), 1 def _des(key, value, flags): if flags == 1: return json.loads(value) return value client = Client((host, port), serializer=_ser, deserializer=_des) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set('key', value) result = client.get('key') tools.assert_equal(result, value)
def test_append_prepend(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) assert result is False result = client.get(b'key') assert result is None result = client.set(b'key', b'value', noreply=False) assert result is True result = client.append(b'key', b'after', noreply=False) assert result is True result = client.get(b'key') assert result == b'valueafter' result = client.prepend(b'key1', b'value', noreply=False) assert result is False result = client.get(b'key1') assert result is None result = client.prepend(b'key', b'before', noreply=False) assert result is True result = client.get(b'key') assert result == b'beforevalueafter'
def append_prepend_test(host, port): client = Client((host, port)) client.flush_all() result = client.append('key', 'value', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.append('key', 'after', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'valueafter') result = client.prepend('key1', 'value', noreply=False) tools.assert_equal(result, False) result = client.get('key1') tools.assert_equal(result, None) result = client.prepend('key', 'before', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key') tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.append(b'key', b'after', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'valueafter') result = client.prepend(b'key1', b'value', noreply=False) tools.assert_equal(result, False) result = client.get(b'key1') tools.assert_equal(result, None) result = client.prepend(b'key', b'before', noreply=False) tools.assert_equal(result, True) result = client.get(b'key') tools.assert_equal(result, b'beforevalueafter')
def get_set_test(host, port): client = Client((host, port)) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set('key', 'value', noreply=False) result = client.get('key') tools.assert_equal(result, 'value') client.set('key2', 'value2', noreply=True) result = client.get('key2') tools.assert_equal(result, 'value2') result = client.get_many(['key', 'key2']) tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
def test_serialization_deserialization(host, port, socket_module): def _ser(key, value): return json.dumps(value).encode('ascii'), 1 def _des(key, value, flags): if flags == 1: return json.loads(value.decode('ascii')) return value client = Client((host, port), serializer=_ser, deserializer=_des, socket_module=socket_module) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set(b'key', value) result = client.get(b'key') assert result == value
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) def get(self, query, params): return self.memcached_client.get(key_serializer(query,params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query,params),document)
def test_gets(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.gets(b'key') assert result == (None, None) result = client.set(b'key', b'value', noreply=False) assert result is True result = client.gets(b'key') assert result[0] == b'value'
def get_set_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set(b'key', b'value', noreply=False) result = client.get(b'key') tools.assert_equal(result, b'value') client.set(b'key2', b'value2', noreply=True) result = client.get(b'key2') tools.assert_equal(result, b'value2') result = client.get_many([b'key', b'key2']) tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'}) result = client.get_many([]) tools.assert_equal(result, {})
def gets_test(host, port): client = Client((host, port)) client.flush_all() result = client.gets('key') tools.assert_equal(result, (None, None)) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.gets('key') tools.assert_equal(result[0], 'value')
def get_set_test(host, port): client = Client((host, port)) client.flush_all() result = client.get('key') tools.assert_equal(result, None) client.set('key', 'value', noreply=False) result = client.get('key') tools.assert_equal(result, 'value') client.set('key2', 'value2', noreply=True) result = client.get('key2') tools.assert_equal(result, 'value2') result = client.get_many(['key', 'key2']) tools.assert_equal(result, {'key': 'value', 'key2': 'value2'}) result = client.get_many([]) tools.assert_equal(result, {})
def test_get_set(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') assert result is None client.set(b'key', b'value', noreply=False) result = client.get(b'key') assert result == b'value' client.set(b'key2', b'value2', noreply=True) result = client.get(b'key2') assert result == b'value2' result = client.get_many([b'key', b'key2']) assert result == {b'key': b'value', b'key2': b'value2'} result = client.get_many([]) assert result == {}
def gets_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.gets(b'key') tools.assert_equal(result, (None, None)) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.gets(b'key') tools.assert_equal(result[0], b'value')
def gets_test(host, port): client = Client((host, port)) client.flush_all() result = client.gets('key') tools.assert_equal(result, (None, None)) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.gets('key') tools.assert_equal(result[0], 'value')
class BackendMemcachedCache() : def __init__(self , host = '192.168.24.138' , port = 11211,timeout=5,ignore_exc=True) : self.rhost = host self.rport = port self.rcli = Client((host, port),timeout=timeout,ignore_exc=ignore_exc) def put(self , k , v) : try : return self.rcli.set(k , v) except BaseException , e : logging.exception(e) return None
def application(environ, start_response): """Do Fun things""" headers = [("Content-Type", "application/vnd.geo+json")] start_response("200 OK", headers) form = parse_formvars(environ) cb = form.get("callback", None) huc12 = form.get("huc12", "000000000000")[:12] mode = form.get("mode", "daily") mckey = "/geojson/huc12_events/%s/%s" % (huc12, mode) mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: res = do(huc12, mode) mc.set(mckey, res, 15) else: res = res.decode("utf-8") mc.close() if cb is not None: res = "%s(%s)" % (cb, res) return [res.encode("ascii")]
class Dao: def __init__(self, host, port): self.db = Client( (host, port), default_noreply=False, serializer=self.__json_serializer, deserializer=self.__json_deserializer ) def __json_serializer(self, key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 def __json_deserializer(self, key, value, flags): if flags == 1: return value.decode('utf-8') if flags == 2: return json.loads(value.decode('utf-8')) raise Exception('Unknown serialization format') def get_count(self): count = self.db.get('count') return int(count) if count else 0 def get_sushi_list(self, count): ids = range(1, count+1) result = self.db.get_multi([f'sushi:{id}' for id in ids]) return [{'id': id, **result[f'sushi:{id}']} for id in ids] def incr_count(self): count = self.db.get('count') if count: return self.db.incr('count', 1) else: self.db.set('count', 1) return 1 def set_sushi(self, sushi): self.db.set('sushi:'+str(sushi['id']), sushi)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): try: from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) except ImportError: raise DependencyError("Error importing pymemcache library for OnionMemcached") def get(self, query, params): return self.memcached_client.get(key_serializer(query,params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query,params),document)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) def get(self, query, params): return self.memcached_client.get(key_serializer(query, params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query, params), document)
def main(environ): """Do something fun""" form = parse_formvars(environ) year = form.get("year", 2015) month = form.get("month", 5) day = form.get("day", 5) year2 = form.get("year2", year) month2 = form.get("month2", month) day2 = form.get("day2", day) scenario = int(form.get("scenario", 0)) v = form.get("v", "avg_loss") huc = form.get("huc") ts = datetime.date(int(year), int(month), int(day)) ts2 = datetime.date(int(year2), int(month2), int(day2)) mckey = "/auto/map.py/%s/%s/%s/%s/%s" % ( huc, ts.strftime("%Y%m%d"), ts2.strftime("%Y%m%d"), scenario, v, ) if form.get("overview"): mckey = "/auto/map.py/%s/%s" % (huc, form.get("zoom")) mc = Client(["iem-memcached", 11211]) res = mc.get(mckey) if res is None: # Lazy import to help speed things up if form.get("overview"): res, do_cache = make_overviewmap(form) else: res, do_cache = make_map(huc, ts, ts2, scenario, v, form) sys.stderr.write("Setting cache: %s\n" % (mckey,)) if do_cache: mc.set(mckey, res, 3600) mc.close() return res
def delete_test(host, port): client = Client((host, port)) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.delete('key', noreply=False) tools.assert_equal(result, 'DELETED') result = client.get('key') tools.assert_equal(result, None)
def delete_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.delete('key', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, None)
def test_delete(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.delete(b'key', noreply=False) assert result is False result = client.get(b'key') assert result is None result = client.set(b'key', b'value', noreply=False) assert result is True result = client.delete(b'key', noreply=False) assert result is True result = client.get(b'key') assert result is None
def delete_test(host, port): client = Client((host, port)) client.flush_all() result = client.delete('key', noreply=False) tools.assert_equal(result, False) result = client.get('key') tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.delete('key', noreply=False) tools.assert_equal(result, True) result = client.get('key') tools.assert_equal(result, None)
class OnionMemcached(OnionCache): def __init__(self, host=('localhost', 11211)): try: from pymemcache.client import Client self.memcached_client = Client(host, serializer=json_serializer, deserializer=json_deserializer) except ImportError: raise DependencyError( "Error importing pymemcache library for OnionMemcached") def get(self, query, params): return self.memcached_client.get(key_serializer(query, params)) def set(self, query, params, document): return self.memcached_client.set(key_serializer(query, params), document)
class PyMemcachedCache(BaseCache): """A cache client based on pymemcache. implemented by pure python and support noreply. """ def __init__(self, config): BaseCache.__init__(self, config) self._client = PyMemcachedClient((config['host'], config['port']), serializer=python_memcache_serializer, deserializer=python_memcache_deserializer, connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT, key_prefix=config.get('key_prefix', '')) def get(self, key): return self._client.get(key) def delete(self, key, noreply=False): self._client.delete(key, noreply) return True def get_many(self, keys): return self._client.get_many(keys) def set(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set(key, value, timeout, noreply) def add(self, key, value, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.add(key, value, timeout, noreply) def set_many(self, data, timeout=None, noreply=False): if timeout is None: timeout = self.default_timeout return self._client.set_many(data, timeout, noreply) def delete_many(self, keys, noreply=False): return self._client.delete_many(keys, noreply) def clear(self): return self._client.flush_all(noreply=False) def incr(self, key, delta=1, noreply=False): return self._client.incr(key, delta, noreply) def decr(self, key, delta=1, noreply=False): return self._client.decr(key, delta, noreply)
def cas_test(host, port): client = Client((host, port)) client.flush_all() result = client.cas('key', 'value', '1', noreply=False) tools.assert_equal(result, None) result = client.set('key', 'value', noreply=False) tools.assert_equal(result, True) result = client.cas('key', 'value', '1', noreply=False) tools.assert_equal(result, False) result, cas = client.gets('key') tools.assert_equal(result, 'value') result = client.cas('key', 'value1', cas, noreply=False) tools.assert_equal(result, True) result = client.cas('key', 'value2', cas, noreply=False) tools.assert_equal(result, False)
def cas_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.cas(b'key', b'value', b'1', noreply=False) tools.assert_equal(result, None) result = client.set(b'key', b'value', noreply=False) tools.assert_equal(result, True) result = client.cas(b'key', b'value', b'1', noreply=False) tools.assert_equal(result, False) result, cas = client.gets(b'key') tools.assert_equal(result, b'value') result = client.cas(b'key', b'value1', cas, noreply=False) tools.assert_equal(result, True) result = client.cas(b'key', b'value2', cas, noreply=False) tools.assert_equal(result, False)
def test_cas(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.cas(b'key', b'value', b'1', noreply=False) assert result is None result = client.set(b'key', b'value', noreply=False) assert result is True result = client.cas(b'key', b'value', b'1', noreply=False) assert result is False result, cas = client.gets(b'key') assert result == b'value' result = client.cas(b'key', b'value1', cas, noreply=False) assert result is True result = client.cas(b'key', b'value2', cas, noreply=False) assert result is False
def cas_test(host, port): client = Client((host, port)) client.flush_all() result = client.cas('key', 'value', '1', noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.set('key', 'value', noreply=False) tools.assert_equal(result, 'STORED') result = client.cas('key', 'value', '1', noreply=False) tools.assert_equal(result, 'EXISTS') result, cas = client.gets('key') tools.assert_equal(result, 'value') result = client.cas('key', 'value1', cas, noreply=False) tools.assert_equal(result, 'STORED') result = client.cas('key', 'value2', cas, noreply=False) tools.assert_equal(result, 'EXISTS')
def incr_decr_test(host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, None) result = client.set(b'key', b'0', noreply=False) tools.assert_equal(result, True) result = client.incr(b'key', 1, noreply=False) tools.assert_equal(result, 1) def _bad_int(): client.incr(b'key', b'foobar') tools.assert_raises(MemcacheClientError, _bad_int) result = client.decr(b'key1', 1, noreply=False) tools.assert_equal(result, None) result = client.decr(b'key', 1, noreply=False) tools.assert_equal(result, 0) result = client.get(b'key') tools.assert_equal(result, b'0')
def incr_decr_test(host, port): client = Client((host, port)) client.flush_all() result = client.incr('key', 1, noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.set('key', '0', noreply=False) tools.assert_equal(result, 'STORED') result = client.incr('key', 1, noreply=False) tools.assert_equal(result, 1) def _bad_int(): client.incr('key', 'foobar') tools.assert_raises(MemcacheClientError, _bad_int) result = client.decr('key1', 1, noreply=False) tools.assert_equal(result, 'NOT_FOUND') result = client.decr('key', 1, noreply=False) tools.assert_equal(result, 0) result = client.get('key') tools.assert_equal(result, '0')
import requests import re from pymemcache.client import Client # User defined variables feedaddr = 'https://feodotracker.abuse.ch/blocklist.php?download=ipblocklist' feedID = 'feodotracker' mkey = 'fetcher_feodoIP:feeddata' killchain = 'Command & Control' # No user modifications needed below. client = Client(('localhost', 11211)) result = client.get(mkey) if isinstance(result, str): splitlines = result.split('\n') else: r = requests.get(feedaddr) client.set(mkey, r.content, expire=7200) splitlines = r.content.split('\n') print('ipv4,feedID,killchain,description') for x in splitlines: x = x.strip() if re.search('^#', x): continue if len(x) == 0: continue print("%s,%s,%s,%s" % (x, feedID, killchain, 'Feodo C&C'))
## NS Notifier userkey (will come from url/cli parameter in the future) try: userkey = settings.userkey except AttributeError: userkey = 1 ## Are we planned to run? (E.g., not disabled through web) try: should_run = mc.get('nsapi_run') except: should_run = True if should_run == None: should_run = True #logger.info('no run tuple in memcache, creating') mc.set('nsapi_run', should_run) #print('should run? ' + str(should_run)) logger.debug('Should run: ' + str(should_run)) if not should_run: sys.exit(0) errors = [] nsapi = ns_api.NSAPI(settings.username, settings.apikey) ## Get the list of stations stations = get_stations(mc) ## Get the current disruptions (globally)