def __init__(self, name, mc_host, mc_port, run=None, run_id=None): self.name = name self.run = run self.run_id = run_id self.mc_host = mc_host self.mc_port = mc_port self.mc_client = Client((self.mc_host, self.mc_port))
def UpdateClassifier(): print("Update Classifier") clf = LogisticRegression( random_state=0, solver='lbfgs', multi_class='multinomial') smt = SMOTETomek(random_state=42) discussions = list(Discussion.objects.filter(reviewed=True)) globalFeaturesIndex = ef.GetGlobalFeaturesIndex( discussions, list(range(0, len(discussions))), ef.E19) X, y = [], [] for discussion in discussions: featureVector = ef.ExtractFeatureFromCorpus( globalFeaturesIndex, discussion.content, ef.E19) X.append(featureVector) y.append(discussion.tag.tag_id) X, y = smt.fit_sample(X, y) selector = GenericUnivariateSelect(chi2, 'percentile', param=20) X = selector.fit_transform(X, y) try: clf.fit(X, y) print("fit done") client = Client(('localhost', 11211)) model = (clf, globalFeaturesIndex, selector) client.set('model', pickle.dumps(model)) model_in_bytes = client.get('model') model_from_cache = pickle.loads(model_in_bytes) print(len(model_from_cache)) except: print("clf failed to cache...")
class MemcachedWrapper(): MEMCACHED_CONNECT_TIMEOUT = 5 # 5 seconds MEMCACHED_TIMEOUT = 5 # 5 seconds def __init__(self, memcached_location): memcached_location_portions = memcached_location.split(":") if len(memcached_location_portions) != 2: raise ValueError( f"Found incorrectly formatted parameter memcached location: {memcached_location}" ) memcached_host = memcached_location_portions[0] memcached_port = int(memcached_location_portions[1]) self.memcached_client = Client( server=(memcached_host, memcached_port), serializer=serde.python_memcache_serializer, deserializer=serde.python_memcache_deserializer, connect_timeout=MemcachedWrapper.MEMCACHED_CONNECT_TIMEOUT, timeout=MemcachedWrapper.MEMCACHED_TIMEOUT) def get(self, key): return self.memcached_client.get(key) def set(self, key, value, timeout=None): self.memcached_client.set(key, value, timeout) def delete(self, key): self.memcached_client.delete(key)
class MyMemcache(object): def __init__(self, ip='localhost', port=11211): # json_serializer(key, value) def json_serializer(key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 # json_deserializer(key, value, flags) python3 def json_deserializer(key, value, flags): if flags == 1: return value.decode('utf-8') if flags == 2: return json.loads(value.decode('utf-8')) raise Exception("Unknown serialization format") self.ip = ip self.port = port self.client = Client((ip, port), serializer=json_serializer, deserializer=json_deserializer, key_prefix='', encoding='utf8', allow_unicode_keys=True) ''' set_task_result ''' @catch_exception def set_value(self, key, value): # set(key, value, expire=0, noreply=None, flags=None) # result = "-".join(result.split()) expire_second = int(60 * 60 * 24 * 1.2) # expire_second must int self.client.set(key=key, value=value, expire=expire_second) # debug_p('[set_value]', key, value) ''' get_task_result ''' @catch_exception def get_value(self, key, default=''): result = self.client.get(key=key, default=default) debug_p('[get_value]', key, result) return result ''' close ''' def client_close(self): try: self.client.close() except Exception as e: # pass
def make_client_pool(self, hostname, mock_socket_values, serializer=None, **kwargs): mock_client = Client(hostname, serializer=serializer, **kwargs) mock_client.sock = MockSocket(mock_socket_values) client = PooledClient(hostname, serializer=serializer) client.client_pool = pool.ObjectPool(lambda: mock_client) return mock_client
def __init__(self, **kwargs): self.client = Client( server=(kwargs.get('address'), int(kwargs.get('port'))), connect_timeout=kwargs.get('connect_timeout', self.DEFAULT_TIMEOUT), timeout=kwargs.get('timeout', self.DEFAULT_TIMEOUT), ignore_exc=kwargs.get('ignore_exc', True))
def startClient(id): global active_threads global max_active_threads try: client = Client(('localhost', 8888)) print('Client ' + str(id) + ' connected') start = time.time() lock.acquire() active_threads += 1 max_active_threads = max(max_active_threads, active_threads) lock.release() for _ in range(500): key, value = randomString(5), randomString(5) client.set(key, value) result = client.get(key) lock.acquire() active_threads -= 1 lock.release() end = time.time() print('Client ' + str(id) + ' exiting. Time spent: ' + str(end - start)) end = time.time() except BaseException as e: print('Exception' + str(e)) return
class MemcachedWrapper(object): '''Class to allow readonly access to underlying memcached connection''' def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError( 'Memcached wrapper improperly configured. Valid memcached host is required!' ) self.__con = Client((host, port)) self._counter = counter('') def __del__(self): self.__con.quit() def get(self, key): return self.__con.get(key) def json(self, key): return pyjson.loads(self.get(key)) def stats(self, extra_keys=[]): data = self.__con.stats() ret = {} for key in data: if key in COUNTER_KEYS: ret['{}_per_sec'.format(key.replace('total_', ''))] = \ round(self._counter.key(key).per_second(data.get(key, 0)), 2) elif key in VALUE_KEYS: ret[key] = data[key] elif key in extra_keys: ret[key] = data[key] return ret
def runOTX(): '''Retrieve intel from OTXv2 API.''' days = int(parser.get('otx', 'days_of_history')) key = parser.get('otx', 'api_key') mem_host = parser.get('memcached', 'mem_host') mem_port = int(parser.get('memcached', 'mem_port')) memcached = Client((mem_host, mem_port)) memcached_agetime = int(parser.get('memcached', 'agetime')) memcached_sleeptime = int(parser.get('memcached', 'sleeptime')) mtime = (datetime.now() - timedelta(days=days)).isoformat() for pulse in iter_pulses(key, mtime): pulse_name = pulse['name'] pulse_id = pulse['id'] for indicator in pulse[u'indicators']: ioc = indicator['indicator'] ioc_type = map_indicator_type(indicator[u'type']) tag = pulse_name + '-' + pulse_id if ioc_type is None: continue try: url = pulse[u'references'][0] except IndexError: url = 'https://otx.alienvault.com' memcached_key = ioc_type + '-' + ioc try: memcached.set(memcached_key.encode('utf-8'), tag.encode('utf-8'), memcached_agetime) except: pass time.sleep(memcached_sleeptime)
def test_incr_decr(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) assert result is None result = client.set(b'key', b'0', noreply=False) assert result is True result = client.incr(b'key', 1, noreply=False) assert result == 1 def _bad_int(): client.incr(b'key', b'foobar') with pytest.raises(MemcacheClientError): _bad_int() result = client.decr(b'key1', 1, noreply=False) assert result is None result = client.decr(b'key', 1, noreply=False) assert result == 0 result = client.get(b'key') assert result == b'0'
def __init__(self, pozyx_offset, lencoder, rencoder, wheel_width=150.5): super(State, self).__init__() self.lencoder = lencoder self.rencoder = rencoder self.wheel_width = wheel_width self.pozyx_offset = pozyx_offset self._state = np.zeros(8) # https://github.com/rlabbe/filterpy self.KF = KalmanFilter(dim_x=8, dim_z=8) self.KF.x = np.array([ 0, ] * 8).T # Will be defined each iteration # self.KF.F = # self.KF.Q = self.KF.H = np.eye(8) # No measurements need to be transformed self.KF.P = np.eye(8) * 500 R_vel = 10 * (self.rencoder.step + self.lencoder.step) R_accel = 100 # 2*10 self.KF.R = np.square( np.diag([ 150 / 2, R_vel, R_accel, 30 / 2, R_vel, R_accel, np.pi / 180, R_vel / (np.pi * self.wheel_width) ])) self.lock = threading.Lock() self.daemon = True self.stop = False self.memcached_client = Client(('localhost', 11211)) self._heading = None self._location = None
def __setstate__(self, state): self.name = state['name'] self.run = state['run'] self.run_id = state['run_id'] self.mc_host = state['mc_host'] self.mc_port = state['mc_port'] self.mc_client = Client((self.mc_host, self.mc_port))
def __init__(self): cf = configparser.ConfigParser() cf.read("./config/config.ini") # elasticache settings # simcomlbs.0rxi5f.cfg.cnw1.cache.amazonaws.com.cn:11211 if "memcachedendpoint" in os.environ: memcachedendpoint = os.environ['memcachedendpoint'] else: memcachedendpoint = cf.get("SimcomLBSMemcached", "memcachedServer") if "memcachedport" in os.environ: memcachedport = os.environ['memcachedport'] else: memcachedport = cf.get("SimcomLBSMemcached", "memcachedServer") logger.info("%s : %s" % (memcachedendpoint, memcachedport)) #-------------Hash client for AWS memcached ------------------------------------------------------ # elasticache_config_endpoint = endpoint + ":" + port # nodes = elasticache_auto_discovery.discover(elasticache_config_endpoint) # nodes = map(lambda x: (x[1], int(x[2])), nodes) # self.memcache_client = HashClient(nodes) #------------------------------------------------------------------------------------------ # ----------------Directlly connected to pod memecached------------------------------------ self.memcache_client = Client(memcachedendpoint, memcachedport) self.memcache_client.set(self, 'abcd123', "1234ab")
class MemcachedCache_Master(MemcachedCache, object): """ the master instance of the memcached cache interface. copies of this object will NOT share the connection, but instantiate their own. When this master is destroyed it can flush the database, effectively erasing all data Parameters ---------- memc_params : dict dictionary with keys [host, port] to the memcached-server instance wait_on_insert : bool if True, wait on insert operations to successfully complete before continuing (default: False) val_ttl : int >0 the time to life for each value in seconds flush_on_del : bool flush the databases when the object is destroyed (default: False) """ def __init__(self, memc_params=_DEFAULT_MEMCACHE_CRED, wait_on_insert=False, val_ttl=12, flush_on_del=False): self.is_master = True self.wait_on_insert = wait_on_insert self.memc_params = memc_params self.val_ttl = val_ttl self.flush_on_del = flush_on_del self.client = None def __del__(self): if self.flush_on_del: self.client = Client((self.memc_params['host'], self.memc_params['port']), default_noreply=not self.wait_on_insert) self.client.flushdb()
class Reader(BaseReader): """ Memcached settings Reader A simple memcached getter using pymemcache library. """ _default_conf = { 'host': 'localhost', 'port': 11211 } def __init__(self, conf): super(Reader, self).__init__(conf) self.client = Client((self.conf['host'], self.conf['port'])) def _get(self, key): result = self.client.get(key) if isinstance(result, six.binary_type): result = result.decode('utf-8') return result def _set(self, key, value): self.client.set(key, value, noreply=False)
class CHIDriverMemcache(CHIDriver): """Драйвер Memcache.""" def __init__(self, *av, **kw): """Конструктор.""" super().__init__(*av, **kw) self.client = Memcache( (self.server[0]["host"], self.server[0]["port"]), connect_timeout=self.connect_timeout, timeout=self.request_timeout, ) def driver_set(self, key, packed_chi_object, ttl): """Переопределение для установки драйвера - в редисе используются две команды, а тут - одна.""" self.client.set(key, packed_chi_object, ttl) def keys(self, mask): """Возвращает ключи по маске.""" raise CHIMethodIsNotSupportedException( "Метод keys для мемкеша не поддерживается.") def erase(self, mask): """Удаление ключей по маске.""" raise CHIMethodIsNotSupportedException( "Метод erase для мемкеша не поддерживается.")
def __init__(self, mc_host, mc_port): self.mc_host = mc_host self.mc_port = mc_port self.client = Client((mc_host, mc_port)) self.setupMonitoringVariables() self.calculateInitialScore()
def test_socket_connect_unix(self): server = '/tmp/pymemcache.{pid}'.format(pid=os.getpid()) with MockUnixSocketServer(server): client = Client(server) client._connect() assert client.sock.family == socket.AF_UNIX
def test_socket_connect_unix(self): server = '/tmp/pymemcache.{pid}'.format(pid=os.getpid()) with MockUnixSocketServer(server): client = Client(server) client._connect() assert client.sock.family == socket.AF_UNIX
def run(self): client = Client(('127.0.0.1', 11211)) url = 'https://talosintelligence.com/documents/ip-blacklist' talosvalue = 'talos-ip' try: response = requests.get(url) if (response): responseArr = [] for line in response.text.splitlines(): if (line.startswith('#') == False): responseArr.append(line) valueCheck = client.get_many(responseArr) for k in responseArr: valueArr = [] tempArr = [] tempArr.append(talosvalue) if k in valueCheck: val = valueCheck[k].decode() valueArr = talosToMemcache.stringHelper(val) for item in valueArr: if item not in tempArr: tempArr.append(item) client.set(k, tempArr, 300) except Exception as e: with open('/var/log/misppullLog.txt', 'a') as file: file.write( '{0} - talosFeed-script failed with error: {1} \n'.format( str(time.asctime()), str(e)))
class MemcacheCli(object): def __init__(self, host, port): self.host = host self.port = port self.client = Client((self.host, self.port)) def set(self, key, value, expire): try: return self.client.set(key, value, expire) except Exception as e: return False def get(self, key): try: return self.client.get(key, default=None) except Exception as e: return None def mset(self, values, expire): try: return self.client.set_many(values, expire) except Exception as e: return False def mget(self, keys): try: return self.client.get_many(keys) except Exception as e: return None
def main(): line1 = Line(1, 5) line2 = Line(2, 6) line3 = Line(6, 8) line4 = Line(7, 9) assert do_intersect(line1, line2) == True assert do_intersect(line1, line3) == False assert do_intersect(line2, line1) == True assert do_intersect(line3, line1) == False assert do_intersect(line3, line4) == True assert compare_versions("1.1", "1.2") == -1 assert compare_versions("1.2.1", "1.2") == 1 assert compare_versions("1.2.1", "1.2.1a") == -1 assert compare_versions("1.2.1b", "1.2.1a") == 1 assert compare_versions("1.2.1", "1.21") == -1 assert compare_versions("2.1.1", "1.2.1") == 1 assert compare_versions("1.2.1", "1.2.1") == 0 assert compare_versions("1.", "1") == 0 assert compare_versions("a1", "1a") == 1 assert compare_versions("1.2.1", "1.19.0") == -1 client = Client('127.0.0.1', 11211) client.set('some_key', 'some_value', expire = 30) assert client.get('some_key') == 'some_value' print("OK")
class Client: def __init__(self, config): self.config = config self.memcache_client = PymemcacheClient(server=(self.config["ip"], self.config["port"]), serializer=self.json_serializer, deserializer=self.json_deserializer, connect_timeout=self.config["connect_timeout"], timeout=self.config["timeout"]) def json_serializer(self, key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 def json_deserializer(self, key, value, flags): if flags == 1: return value if flags == 2: return json.loads(value) raise Exception("Unknown serialization format") def write(self, key, message): logger.info("Writing to cache: {}".format(key)) self.memcache_client.set(key, message, expire=self.config["key_expiration"], noreply=self.config["noreply_flag"]) def read(self, key): logger.info("Reading from cache: {}".format(key)) return self.memcache_client.get(key)
class MemcachedWrapper(object): '''Class to allow readonly access to underlying memcached connection''' def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError('Memcached wrapper improperly configured. Valid memcached host is required!') self.__con = Client((host, port)) self._counter = counter('') def __del__(self): self.__con.quit() def get(self, key): return self.__con.get(key) def json(self, key): return pyjson.loads(self.get(key)) def stats(self, extra_keys=[]): data = self.__con.stats() ret = {} for key in data: if key in COUNTER_KEYS: ret['{}_per_sec'.format(key.replace('total_', ''))] = \ round(self._counter.key(key).per_second(data.get(key, 0)), 2) elif key in VALUE_KEYS: ret[key] = data[key] elif key in extra_keys: ret[key] = data[key] return ret
class BaseModel(object): def __init__(self): self.conn = pymysql.connect(host=CONFIG['DB_HOST'], port=CONFIG['DB_PORT'], user=CONFIG['DB_USER'], password=CONFIG['DB_PASSWORD'], db=CONFIG['DATABASE'], charset='utf8mb4') if CONFIG['MEMCACHED_ADDRESS'] and CONFIG['MEMCACHED_PORT']: host, port = CONFIG['MEMCACHED_ADDRESS'], CONFIG['MEMCACHED_PORT'] self.cache = Client( (host, port), timeout=CONFIG['MEMCACHED_TIMEOUT'], deserializer=lambda k, v, f: str(v, encoding='utf-8') if isinstance(v, bytes) else v) self.cache_expire = CONFIG['MEMCACHED_CACHE_EXPIRE'] def __del__(self): """ close connection :return: None """ if hasattr(self, 'cache'): self.cache.close() if hasattr(self, 'conn'): self.conn.close()
class NoSQLCache: __instance = None timeout = 5 @staticmethod def inst(): if NoSQLCache.__instance == None: NoSQLCache.__instance = NoSQLCache(SqliteDb.inst()) return NoSQLCache.__instance def __init__(self, sql_db_object): self.client = Client(('localhost', 11211)) self.sql_db = sql_db_object def get(self, key): key_corrector = re.sub(r' ', '$', key) value = self.client.get(key_corrector) if value: return eval(value.decode('utf-8')) else: value = self.sql_db.get(key) self.set(key_corrector, value, self.timeout) return value def set(self, key, value, timeout): self.client.set(key, value, expire=timeout)
def execqueryWithMem(name, c1, c2): try: mysql_conn = mysql.connector.connect(host=host, user=dbusername, password=dbpassword, database=dbname, port=port) mc = Client(('cloudassgn.hw0lsb.0001.use2.cache.amazonaws.com', 11211)) query = 'select ' + name + ' from Earthquake where ' + c1 + ' and ' + c2 hashVal = hashlib.sha224(query).hexdigest() starttime = int(round(time.time() * 1000)) data = mc.get(hashVal) count = 0 if not data: for i in range(0, 250): cursor = mysql_conn.cursor() cursor.execute(query) row = cursor.fetchall() count = cursor.rowcount cursor.close() mc.set(hashVal, count) endtime = int(round(time.time() * 1000)) totalexectime = endtime - starttime mysql_conn.close() resultStr = '<div style="font-size:14px;margin-top: 30px;"><div> Last Name : Manakan </div><div> Last 4 digit ID : 6131 </div><div> Class Section : 10:30 AM </div></div>' resultStr = resultStr + '<br> Time taken : ' + str(totalexectime) + ' msecs' resultStr = resultStr + '<br> Rows effected : ' + str(count) return resultStr except Exception as e: print e return 'Error ' + str(e)
def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError( 'Memcached wrapper improperly configured. Valid memcached host is required!' ) self.__con = Client((host, port)) self._counter = counter('')
def __init__(self, log_): try: self.log = log_ self.client = Client(('localhost', 11211), serializer=self.json_serializer, deserializer=self.json_deserializer) except Exception as e: log_.error('{0}'.format(e.message))
def set_test(request): client = Client(('/opt/www/memcached/memcached.sock'), serializer=json_serializer, deserializer=json_deserializer) client.set('new', {'status': 'updated'}) # result = client.get('key') return JsonResponse({'status': 'ok'})
def adduser(request): if request.method == 'POST': data = request.get_data() json_data = json.loads(data.decode('utf-8')) username = json_data.get('username') password = json_data.get('password') email = json_data.get('email') userid = ''.join([ random.choice(string.ascii_letters + string.digits) for i in range(16) ]) if username is None or password is None or email is None: return json.dumps({'status': 'error', "error": "have empty block"}) key = ''.join([ random.choice(string.ascii_letters + string.digits) for i in range(16) ]) timestamp = int(time.time()) res = dbcontroller.mysql_checkuser(username, email) if res == False: return json.dumps({ 'status': 'error', 'error': 'email already register or username already in use' }) #use memcache mc = Client((memhost, memport)) info = { "username": username, "password": password, "email": email, "timestamp": timestamp, "key": key, "userid": userid } print(info) mc.set(email, info) #return json.dumps({'status':mc.get(email)}) if sendMail(email, key) == True: return json.dumps({'status': 'OK'}) else: return json.dumps({ 'status': 'error', 'error': 'invalid email address' }) else: return json.dumps({'status': 'error'})
class Memcached(object): """Memcached based caching. This is extremely lightweight in terms of memory usage, but it appears to be slightly slower than local database caching, and a lot slower than local memory based caching. This should be used if the cache is to be shared between multiple users. """ LRU = LeastRecentlyUsed = 0 def __init__(self, url='127.0.0.1', mode=LRU, ttl=None): """Create a new engine. Parameters: url (str): Memcached server to connect to. mode (int): How to purge the old keys. This does not affect anything as memcached is LRU only. The option is there to match the other engines. ttl (int): Time the cache is valid for. Set to None or 0 for infinite. """ try: from pymemcache.client.base import Client self.client = Client(url, timeout=3, connect_timeout=3) except ImportError: from memcache import Client self.client = Client([url], socket_timeout=3) self.ttl = ttl # Don't allow cross version caches # Pickle may have incompatibilities between versions self._key_prefix = '{}.{}.'.format(sys.version_info.major, sys.version_info.minor) def get(self, key): """Get the value belonging to a key. An error will be raised if the cache is expired or doesn't exist. """ value = self.client.get(self._key_prefix + key) if value is None: raise exceptions.CacheNotFound(key) return _decode(value) def put(self, key, value, ttl=None): """Write a new value to the cache. This will overwrite any old cache with the same key. """ if ttl is None: ttl = self.ttl self.client.set(self._key_prefix + key, _encode(value), ttl or 0, noreply=True) def delete(self, key): """Delete an item of cache if it exists.""" return self.client.delete(self._key_prefix + key, noreply=False)
def plugin(srv, item): srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target) if HAVE_SLACK == False: srv.logging.error("slacker module missing") return False token = item.config.get("token") if token is None: srv.logging.error("No token found for slack") return False try: channel_id, size_x, size_y, timespan, dir_to_save = item.addrs except: srv.logging.error("Incorrect target configuration") return False # make animated gif, save to local disk, upload to slack channel client = Client(("127.0.0.1", 11211)) images_original = [] images_resized = [] size = size_x, size_y cur_month = datetime.datetime.now().strftime("%Y%m") cur_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") cur_dir = dir_to_save + "/" + cur_month if not os.path.exists(cur_dir): os.makedirs(cur_dir) cur_img_original = cur_dir + "/" + cur_time + ".gif" cur_img_resized = cur_dir + "/" + cur_time + "_s.gif" # make a gif file try: result = client.get("curno") if result: curno = int(result) for i in range((curno - timespan), curno): data = client.get(str(i)) if data: im = Image.open(StringIO.StringIO(data)) images_original.append(im) imresized = ImageOps.fit(im, size, Image.ANTIALIAS) images_resized.append(imresized) if len(images_original) > 0: images2gif.writeGif(cur_img_original, images_original, duration=0.2) images2gif.writeGif(cur_img_resized, images_resized, duration=0.2) except Exception, e: srv.logging.warning("Cannot make a gif: %s" % (str(e))) return False
def test_socket_close(self): server = ("example.com", 11211) client = Client(server, socket_module=MockSocketModule()) client._connect() assert client.sock is not None client.close() assert client.sock is None
def make_client(self, mock_socket_values, **kwargs): client = Client(None, **kwargs) # mock out client._connect() rather than hard-settting client.sock to # ensure methods are checking whether self.sock is None before # attempting to use it sock = MockSocket(list(mock_socket_values)) client._connect = mock.Mock(side_effect=functools.partial( setattr, client, "sock", sock)) return client
def test_socket_connect_closes_on_failure(self): server = ("example.com", 11211) socket_module = MockSocketModule(connect_failure=OSError()) client = Client(server, socket_module=socket_module) with pytest.raises(OSError): client._connect() assert len(socket_module.sockets) == 1 assert socket_module.sockets[0].connections == [] assert socket_module.sockets[0].closed
def test_socket_close_exception(self): server = ("example.com", 11211) socket_module = MockSocketModule(close_failure=OSError()) client = Client(server, socket_module=socket_module) client._connect() assert client.sock is not None client.close() assert client.sock is None
def test_incr_decr(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) assert result is None result = client.set(b'key', b'0', noreply=False) assert result is True result = client.incr(b'key', 1, noreply=False) assert result == 1 def _bad_int(): client.incr(b'key', b'foobar') with pytest.raises(MemcacheClientError): _bad_int() result = client.decr(b'key1', 1, noreply=False) assert result is None result = client.decr(b'key', 1, noreply=False) assert result == 0 result = client.get(b'key') assert result == b'0'
class MemcachedCache(CachualCache): """A cache using `Memcached <https://memcached.org/>`_ as the backing cache. The same caveats apply to keys and values as for Redis - you should only try to store strings (using the packing/unpacking functions). See the documentation on Keys and Values here: :class:`pymemcache.client.base.Client`. :type host: string :param host: The Memcached host to use for the cache. :type port: integer :param port: The port to use for the Memcached server. :type kwargs: dict :param kwargs: Any additional args to pass to the :class:`CachualCache` constructor. """ def __init__(self, host='localhost', port=11211, **kwargs): super(MemcachedCache, self).__init__(**kwargs) self.client = MemcachedClient((host, port)) def get(self, key): """Get a value from the cache using the given key. :type key: string :param key: The cache key to get the value for. :returns: The value for the cache key, or None in the case of cache miss. """ return self.client.get(key) def put(self, key, value, ttl=None): """Put a value into the cache at the given key. For constraints on keys and values, see :class:`pymemcache.client.base.Client`. :type key: string :param key: The cache key to use for the value. :param value: The value to store in the cache. :type ttl: integer :param ttl: The time-to-live for key in seconds, after which it will expire. """ if ttl is None: ttl = 0 self.client.set(key, value, expire=ttl)
class Memcached(object): DELAY = 0.5 DEBUG = False def __init__(self, hostname, port, **params): self.mc = Client((hostname, port)) def handle(self, topic, message): """ """ if 'cmd' not in message: raise Exception("Bad message: no command") cmd = message['cmd'] if not hasattr(self, cmd): raise Exception("Unknown command: " + cmd) tryit = True while tryit: tryit = False try: getattr(self, cmd)(message) except MemcacheUnexpectedCloseError: # Server dropped dead - we'll retry tryit = True except IOError: # Something network-related - retry tryit = True if tryit: time.sleep(self.DELAY) def set(self, message): text = message['val'].encode('utf-8') if message.get('sbt', None): purge_time = time.time() + message.get('uto', 0) text = text.replace('$UNIXTIME$', '%.6f' % purge_time) if self.DEBUG: print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'), text, int(message['ttl']))) self.mc.set(message['key'].encode('utf-8'), text, int(message['ttl'])) def delete(self, message): self.mc.delete(message['key'])
def __init__(self, timeline_rate_reserve=5, multi_proc_logger=None): """ The reserve arguments are how many requests to hold back on to leave some form of buffer in place with regards to API limits """ if multi_proc_logger: self.logger = multi_proc_logger else: self.logger = utils.MultiProcessCheckingLogger(module_logger) self.memcacheClient = MemCacheClient( (interns_settings.memcache_host, interns_settings.memcache_port) ) self.timeline_rate_reserve = timeline_rate_reserve self.tl_total_reqs = interns_settings.twitter_timeline_requests self.tl_reqs_left = interns_settings.twitter_timeline_req_left self.tl_reqs_reset_time = interns_settings.twitter_timeline_reset_time self.update_limits()
def test_serialization_deserialization(host, port, socket_module): def _ser(key, value): return json.dumps(value).encode('ascii'), 1 def _des(key, value, flags): if flags == 1: return json.loads(value.decode('ascii')) return value client = Client((host, port), serializer=_ser, deserializer=_des, socket_module=socket_module) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set(b'key', value) result = client.get(b'key') assert result == value
def make_client(self, mock_socket_values, **kwargs): client = Client(None, key_prefix=b"xyz:", **kwargs) client.sock = MockSocket(list(mock_socket_values)) return client
def make_client(self, mock_socket_values, **kwargs): mock_client = Client(None, **kwargs) mock_client.sock = MockSocket(list(mock_socket_values)) client = PooledClient(None, **kwargs) client.client_pool = pool.ObjectPool(lambda: mock_client) return client
handleCommand("412") time.sleep(1) # check current input source no if handleCommand("47"): time.sleep(4) m, n = getscreenimage() return m, n #------------------------ #logging.basicConfig( # format='# %(levelname)s: %(message)s', # level=logging.DEBUG, #) client = Client(('127.0.0.1', 11211)) try: result = client.get('appletv') if result: sys.exit("time is not passed") else: print ("appletv is on") # get current channel and change input to HDMI2 # tv should be on m, n = getCHandHDMI2() if m != n: changeinout(m, n) else: handleCommand("20") client.set('appletv', 'true', 60)
def test_socket_connect_unix(self): server = "/tmp/memcache" with MockUnixSocket(server): client = Client(server) assert client._use_unix_socket client._connect()
def test_socket_connect(self): server = ("example.com", 11211) client = Client(server, socket_module=MockSocketModule()) client._connect() assert client.sock.connections == [server] timeout = 2 connect_timeout = 3 client = Client( server, connect_timeout=connect_timeout, timeout=timeout, socket_module=MockSocketModule(), ) client._connect() assert client.sock.timeouts == [connect_timeout, timeout] client = Client(server, socket_module=MockSocketModule()) client._connect() assert client.sock.socket_options == [] client = Client(server, socket_module=MockSocketModule(), no_delay=True) client._connect() assert client.sock.socket_options == [ (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) ]
def make_client(self, values): client = Client(None) client.sock = MockSocket(list(values)) return client
class TwitterLimits(object): """ Checks against either the twitter API or memcached twitter API results for twitter API requests limits """ def __init__(self, timeline_rate_reserve=5, multi_proc_logger=None): """ The reserve arguments are how many requests to hold back on to leave some form of buffer in place with regards to API limits """ if multi_proc_logger: self.logger = multi_proc_logger else: self.logger = utils.MultiProcessCheckingLogger(module_logger) self.memcacheClient = MemCacheClient( (interns_settings.memcache_host, interns_settings.memcache_port) ) self.timeline_rate_reserve = timeline_rate_reserve self.tl_total_reqs = interns_settings.twitter_timeline_requests self.tl_reqs_left = interns_settings.twitter_timeline_req_left self.tl_reqs_reset_time = interns_settings.twitter_timeline_reset_time self.update_limits() def update_limits(self): """ Update the limits associated with the twitter api First attempts to check memcache and then checks directly with the twitter API """ self.logger.debug(__name__, 'Updating twitter limits') cache_total_tl_reqs = self.memcacheClient.get('timeline_limit') if cache_total_tl_reqs: self.tl_total_reqs = int(cache_total_tl_reqs) cache_tl_reqs_left = self.memcacheClient.get('timeline_remaining') if cache_tl_reqs_left: self.tl_reqs_left = int(cache_tl_reqs_left) cache_tl_reqs_reset_time = self.memcacheClient.get( 'timeline_reset' ) if cache_tl_reqs_reset_time: self.tl_reqs_reset_time = int(cache_tl_reqs_reset_time) utc_now = datetime.utcnow() utc_secs = (utc_now - epoch_time).total_seconds() secs_until_reset = self.tl_reqs_reset_time - utc_secs if secs_until_reset <= 0: # Force getting rates from twitter self.tl_reqs_reset_time = None update_values_valid = ( self.tl_total_reqs and (self.tl_reqs_left is not None) and self.tl_reqs_reset_time ) if not update_values_valid: self.logger.debug(__name__, 'Making twitter API limits request') update_vals = ( twitterClient.get_user_timeline_rate_limit() ) self.tl_total_reqs = update_vals.limit self.tl_reqs_left = update_vals.remaining self.tl_reqs_reset_time = update_vals.reset else: self.logger.debug( __name__, 'Using twitter API limits from memcache' ) self.logger.debug( __name__, 'Total twitter timeline requests allowed is: {0}'.format( self.tl_total_reqs ) ) self.logger.debug( __name__, 'Number of twitter timeline requests left is: {0}'.format( self.tl_reqs_left ) ) self.logger.debug( __name__, 'Twitter timeline request reset time is: {0}'.format( self.tl_reqs_reset_time ) ) def get_sleep_between_jobs(self): """ Calculate the sleep time between jobs as to not run into the twitter API limits """ self.update_limits() self.logger.debug(__name__, 'Updating sleep time between jobs') utc_now = datetime.utcnow() utc_secs = (utc_now - epoch_time).total_seconds() self.logger.debug(__name__, 'UTC in seconds {0}'.format(utc_secs)) secs_until_reset = self.tl_reqs_reset_time - utc_secs self.logger.debug( __name__, 'Seconds until API reset {0}'.format(secs_until_reset) ) buffered_tl_reqs_left = ( self.tl_reqs_left - self.timeline_rate_reserve ) self.logger.debug( __name__, 'Buffered timeline requests left {0}'.format( buffered_tl_reqs_left ) ) if buffered_tl_reqs_left <= 0: sleep_time = secs_until_reset else: sleep_time = secs_until_reset / buffered_tl_reqs_left self.logger.debug(__name__, 'Sleep time {0}'.format(sleep_time)) return sleep_time
def __init__(self, host='localhost', port=11211, **kwargs): super(MemcachedCache, self).__init__(**kwargs) self.client = MemcachedClient((host, port))
def test_misc(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all()
def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError('Memcached wrapper improperly configured. Valid memcached host is required!') self.__con = Client((host, port)) self._counter = counter('')
def make_client(self, mock_socket_values, serializer=None): client = Client(None, serializer=serializer) client.sock = MockSocket(list(mock_socket_values)) return client
def make_client(self, mock_socket_values, serializer=None): mock_client = Client(None, serializer=serializer, key_prefix=b'xyz:') mock_client.sock = MockSocket(list(mock_socket_values)) client = PooledClient(None, serializer=serializer, key_prefix=b'xyz:') client.client_pool = pool.ObjectPool(lambda: mock_client) return client
#!/usr/bin/env python import sys import time import urllib2 from PIL import Image from pymemcache.client.base import Client client = Client(('127.0.0.1', 11211)) url = "http://127.0.0.1:5080/index1.jpg" i = 0 try: while True: try: input = urllib2.urlopen(url) input.readline(); input.readline() content_length = int(input.readline().split(": ")[1].strip()) input.readline(); input.readline() data = input.read(content_length) client.set(str(i), data, 30) client.set('curno', i) i = i + 1 time.sleep(0.7) except Exception, e: print e.__doc__ print e.message sys.exit(1) except KeyboardInterrupt:
def __init__(self, conf): super(Reader, self).__init__(conf) self.client = Client((self.conf['host'], self.conf['port']))
def gao_memcache(): client = Client((app.config['MEMCACHE_HOST'], app.config['MEMCACHE_PORT'])) client.set('some_key', 'some_value', expire=10) result = client.get('some_key') print result
def __init__(self, hostname, port, **params): self.mc = Client((hostname, port))