class Cache: client = None __path = "" __md5 = "" __chunks = [] def __init__(self, chunk_size=1000000): self.client = Client(('localhost', 11211)) self.__chunk_size = chunk_size def readFile(self, path, data=None): self.__path = path if data is not None: data = data.encode() self.__md5 = self.__getMd5(data) self.__getChunks(data) self.__cache() return self.__chunks def getFile(self, name): """ TODO: use join() instead """ i = 0 content = bytearray() while True: data = self.client.get("{}:{}".format(name, i)) if not data: break content += data i += 1 checksum = self.client.get("{}:hash".format(name)) new_md5 = hashlib.md5(content).digest() assert checksum == new_md5, 'data corrupted' return content def __cache(self): self.client.set("{}:hash".format(self.__path), self.__md5) for i, chunk in enumerate(self.__chunks): item_key = "{}:{}".format(self.__path, i) self.client.set(item_key, chunk) def __getMd5(self, data=None): if data is not None: return hashlib.md5(data).digest() file = open(self.__path, 'rb') content = file.read() file.close() return hashlib.md5(content).digest() def __getChunks(self, data=None): self.__chunks = [] if data is not None: return hashlib.md5(data).digest() file = open(self.__path, 'rb') count = 0 while True: data = file.read(self.__chunk_size) if not data: break self.__chunks.append(data)
def plugin(srv, item): srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target) if HAVE_SLACK == False: srv.logging.error("slacker module missing") return False token = item.config.get("token") if token is None: srv.logging.error("No token found for slack") return False try: channel_id, size_x, size_y, timespan, dir_to_save = item.addrs except: srv.logging.error("Incorrect target configuration") return False # make animated gif, save to local disk, upload to slack channel client = Client(("127.0.0.1", 11211)) images_original = [] images_resized = [] size = size_x, size_y cur_month = datetime.datetime.now().strftime("%Y%m") cur_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") cur_dir = dir_to_save + "/" + cur_month if not os.path.exists(cur_dir): os.makedirs(cur_dir) cur_img_original = cur_dir + "/" + cur_time + ".gif" cur_img_resized = cur_dir + "/" + cur_time + "_s.gif" # make a gif file try: result = client.get("curno") if result: curno = int(result) for i in range((curno - timespan), curno): data = client.get(str(i)) if data: im = Image.open(StringIO.StringIO(data)) images_original.append(im) imresized = ImageOps.fit(im, size, Image.ANTIALIAS) images_resized.append(imresized) if len(images_original) > 0: images2gif.writeGif(cur_img_original, images_original, duration=0.2) images2gif.writeGif(cur_img_resized, images_resized, duration=0.2) except Exception, e: srv.logging.warning("Cannot make a gif: %s" % (str(e))) return False
def get_from_cache(n): client = Client(('my_memc', 11211)) if client.get(str(n)): result = client.get(str(n)).decode("utf-8") try: return int(result) except ValueError: return None return None
def memcclient(self, host): c = Client(host) lock = c.get('lock') while lock: sleep(.25) lock = c.get('lock') c.set('lock', 'true') try: yield c finally: c.delete('lock')
class Cache(): def __init__(self, hostname=cache_server_host, port=cache_server_port): self.server = Client((hostname, port)) def set(self, key, value, expiry=0): self.server.set(str(key), value, expiry) def get(self, key): if self.server.get(str(key)): value = self.server.get(str(key)).decode("utf-8") return int(value) def delete(self, key): self.server.delete(str(key))
class Reader(BaseReader): """ Memcached settings Reader A simple memcached getter using pymemcache library. """ _default_conf = { 'host': 'localhost', 'port': 11211 } def __init__(self, conf): super(Reader, self).__init__(conf) self.client = Client((self.conf['host'], self.conf['port'])) def _get(self, key): result = self.client.get(key) if isinstance(result, six.binary_type): result = result.decode('utf-8') return result def _set(self, key, value): self.client.set(key, value, noreply=False)
def startClient(id): global active_threads global max_active_threads try: client = Client(('localhost', 8888)) print('Client ' + str(id) + ' connected') start = time.time() lock.acquire() active_threads += 1 max_active_threads = max(max_active_threads, active_threads) lock.release() for _ in range(500): key, value = randomString(5), randomString(5) client.set(key, value) result = client.get(key) lock.acquire() active_threads -= 1 lock.release() end = time.time() print('Client ' + str(id) + ' exiting. Time spent: ' + str(end - start)) end = time.time() except BaseException as e: print('Exception' + str(e)) return
def main(): client = Client(('localhost', 11211)) client.set('some_key', 'some_value') client.set('some_key1', 'some_value') client.set('some_key2', 'some_value') client.set('some_key3', 'some_value') client.set('some_key4', 'some_value') # client.set('some_key1', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # client.set('some_key2', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # client.set('some_key3', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # print (result) result = client.get_multi(['some_key2', 'some_key3']) print(result) print(client.delete('some_key')) print(client.get('some_key')) client.set('new', 'unknown value') client.set('test1', 'tesing unknow reasonable text') client.set('test2', 'although i know it will be hash value') client.set('test3', 'show me the value') client.set('test4', 'whatever it takes') client.set('test5', 'something at works') return True
class MyMemcache(object): def __init__(self, ip='localhost', port=11211): # json_serializer(key, value) def json_serializer(key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 # json_deserializer(key, value, flags) python3 def json_deserializer(key, value, flags): if flags == 1: return value.decode('utf-8') if flags == 2: return json.loads(value.decode('utf-8')) raise Exception("Unknown serialization format") self.ip = ip self.port = port self.client = Client((ip, port), serializer=json_serializer, deserializer=json_deserializer, key_prefix='', encoding='utf8', allow_unicode_keys=True) ''' set_task_result ''' @catch_exception def set_value(self, key, value): # set(key, value, expire=0, noreply=None, flags=None) # result = "-".join(result.split()) expire_second = int(60 * 60 * 24 * 1.2) # expire_second must int self.client.set(key=key, value=value, expire=expire_second) # debug_p('[set_value]', key, value) ''' get_task_result ''' @catch_exception def get_value(self, key, default=''): result = self.client.get(key=key, default=default) debug_p('[get_value]', key, result) return result ''' close ''' def client_close(self): try: self.client.close() except Exception as e: # pass
def test_incr_decr(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() result = client.incr(b'key', 1, noreply=False) assert result is None result = client.set(b'key', b'0', noreply=False) assert result is True result = client.incr(b'key', 1, noreply=False) assert result == 1 def _bad_int(): client.incr(b'key', b'foobar') with pytest.raises(MemcacheClientError): _bad_int() result = client.decr(b'key1', 1, noreply=False) assert result is None result = client.decr(b'key', 1, noreply=False) assert result == 0 result = client.get(b'key') assert result == b'0'
def UpdateClassifier(): print("Update Classifier") clf = LogisticRegression( random_state=0, solver='lbfgs', multi_class='multinomial') smt = SMOTETomek(random_state=42) discussions = list(Discussion.objects.filter(reviewed=True)) globalFeaturesIndex = ef.GetGlobalFeaturesIndex( discussions, list(range(0, len(discussions))), ef.E19) X, y = [], [] for discussion in discussions: featureVector = ef.ExtractFeatureFromCorpus( globalFeaturesIndex, discussion.content, ef.E19) X.append(featureVector) y.append(discussion.tag.tag_id) X, y = smt.fit_sample(X, y) selector = GenericUnivariateSelect(chi2, 'percentile', param=20) X = selector.fit_transform(X, y) try: clf.fit(X, y) print("fit done") client = Client(('localhost', 11211)) model = (clf, globalFeaturesIndex, selector) client.set('model', pickle.dumps(model)) model_in_bytes = client.get('model') model_from_cache = pickle.loads(model_in_bytes) print(len(model_from_cache)) except: print("clf failed to cache...")
class MemcachedWrapper(object): '''Class to allow readonly access to underlying memcached connection''' def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError('Memcached wrapper improperly configured. Valid memcached host is required!') self.__con = Client((host, port)) self._counter = counter('') def __del__(self): self.__con.quit() def get(self, key): return self.__con.get(key) def json(self, key): return pyjson.loads(self.get(key)) def stats(self, extra_keys=[]): data = self.__con.stats() ret = {} for key in data: if key in COUNTER_KEYS: ret['{}_per_sec'.format(key.replace('total_', ''))] = \ round(self._counter.key(key).per_second(data.get(key, 0)), 2) elif key in VALUE_KEYS: ret[key] = data[key] elif key in extra_keys: ret[key] = data[key] return ret
class MemcacheCli(object): def __init__(self, host, port): self.host = host self.port = port self.client = Client((self.host, self.port)) def set(self, key, value, expire): try: return self.client.set(key, value, expire) except Exception as e: return False def get(self, key): try: return self.client.get(key, default=None) except Exception as e: return None def mset(self, values, expire): try: return self.client.set_many(values, expire) except Exception as e: return False def mget(self, keys): try: return self.client.get_many(keys) except Exception as e: return None
class MemcachedWrapper(): MEMCACHED_CONNECT_TIMEOUT = 5 # 5 seconds MEMCACHED_TIMEOUT = 5 # 5 seconds def __init__(self, memcached_location): memcached_location_portions = memcached_location.split(":") if len(memcached_location_portions) != 2: raise ValueError( f"Found incorrectly formatted parameter memcached location: {memcached_location}" ) memcached_host = memcached_location_portions[0] memcached_port = int(memcached_location_portions[1]) self.memcached_client = Client( server=(memcached_host, memcached_port), serializer=serde.python_memcache_serializer, deserializer=serde.python_memcache_deserializer, connect_timeout=MemcachedWrapper.MEMCACHED_CONNECT_TIMEOUT, timeout=MemcachedWrapper.MEMCACHED_TIMEOUT) def get(self, key): return self.memcached_client.get(key) def set(self, key, value, timeout=None): self.memcached_client.set(key, value, timeout) def delete(self, key): self.memcached_client.delete(key)
class Client: def __init__(self, config): self.config = config self.memcache_client = PymemcacheClient(server=(self.config["ip"], self.config["port"]), serializer=self.json_serializer, deserializer=self.json_deserializer, connect_timeout=self.config["connect_timeout"], timeout=self.config["timeout"]) def json_serializer(self, key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 def json_deserializer(self, key, value, flags): if flags == 1: return value if flags == 2: return json.loads(value) raise Exception("Unknown serialization format") def write(self, key, message): logger.info("Writing to cache: {}".format(key)) self.memcache_client.set(key, message, expire=self.config["key_expiration"], noreply=self.config["noreply_flag"]) def read(self, key): logger.info("Reading from cache: {}".format(key)) return self.memcache_client.get(key)
class NoSQLCache: __instance = None timeout = 5 @staticmethod def inst(): if NoSQLCache.__instance == None: NoSQLCache.__instance = NoSQLCache(SqliteDb.inst()) return NoSQLCache.__instance def __init__(self, sql_db_object): self.client = Client(('localhost', 11211)) self.sql_db = sql_db_object def get(self, key): key_corrector = re.sub(r' ', '$', key) value = self.client.get(key_corrector) if value: return eval(value.decode('utf-8')) else: value = self.sql_db.get(key) self.set(key_corrector, value, self.timeout) return value def set(self, key, value, timeout): self.client.set(key, value, expire=timeout)
class MemcachedWrapper(object): '''Class to allow readonly access to underlying memcached connection''' def __init__(self, counter, host, port=11211, socket_connect_timeout=1): if not host: raise ConfigurationError( 'Memcached wrapper improperly configured. Valid memcached host is required!' ) self.__con = Client((host, port)) self._counter = counter('') def __del__(self): self.__con.quit() def get(self, key): return self.__con.get(key) def json(self, key): return pyjson.loads(self.get(key)) def stats(self, extra_keys=[]): data = self.__con.stats() ret = {} for key in data: if key in COUNTER_KEYS: ret['{}_per_sec'.format(key.replace('total_', ''))] = \ round(self._counter.key(key).per_second(data.get(key, 0)), 2) elif key in VALUE_KEYS: ret[key] = data[key] elif key in extra_keys: ret[key] = data[key] return ret
def main(): line1 = Line(1, 5) line2 = Line(2, 6) line3 = Line(6, 8) line4 = Line(7, 9) assert do_intersect(line1, line2) == True assert do_intersect(line1, line3) == False assert do_intersect(line2, line1) == True assert do_intersect(line3, line1) == False assert do_intersect(line3, line4) == True assert compare_versions("1.1", "1.2") == -1 assert compare_versions("1.2.1", "1.2") == 1 assert compare_versions("1.2.1", "1.2.1a") == -1 assert compare_versions("1.2.1b", "1.2.1a") == 1 assert compare_versions("1.2.1", "1.21") == -1 assert compare_versions("2.1.1", "1.2.1") == 1 assert compare_versions("1.2.1", "1.2.1") == 0 assert compare_versions("1.", "1") == 0 assert compare_versions("a1", "1a") == 1 assert compare_versions("1.2.1", "1.19.0") == -1 client = Client('127.0.0.1', 11211) client.set('some_key', 'some_value', expire = 30) assert client.get('some_key') == 'some_value' print("OK")
def execqueryWithMem(name, c1, c2): try: mysql_conn = mysql.connector.connect(host=host, user=dbusername, password=dbpassword, database=dbname, port=port) mc = Client(('cloudassgn.hw0lsb.0001.use2.cache.amazonaws.com', 11211)) query = 'select ' + name + ' from Earthquake where ' + c1 + ' and ' + c2 hashVal = hashlib.sha224(query).hexdigest() starttime = int(round(time.time() * 1000)) data = mc.get(hashVal) count = 0 if not data: for i in range(0, 250): cursor = mysql_conn.cursor() cursor.execute(query) row = cursor.fetchall() count = cursor.rowcount cursor.close() mc.set(hashVal, count) endtime = int(round(time.time() * 1000)) totalexectime = endtime - starttime mysql_conn.close() resultStr = '<div style="font-size:14px;margin-top: 30px;"><div> Last Name : Manakan </div><div> Last 4 digit ID : 6131 </div><div> Class Section : 10:30 AM </div></div>' resultStr = resultStr + '<br> Time taken : ' + str(totalexectime) + ' msecs' resultStr = resultStr + '<br> Rows effected : ' + str(count) return resultStr except Exception as e: print e return 'Error ' + str(e)
class Memcached(object): """Memcached based caching. This is extremely lightweight in terms of memory usage, but it appears to be slightly slower than local database caching, and a lot slower than local memory based caching. This should be used if the cache is to be shared between multiple users. """ LRU = LeastRecentlyUsed = 0 def __init__(self, url='127.0.0.1', mode=LRU, ttl=None): """Create a new engine. Parameters: url (str): Memcached server to connect to. mode (int): How to purge the old keys. This does not affect anything as memcached is LRU only. The option is there to match the other engines. ttl (int): Time the cache is valid for. Set to None or 0 for infinite. """ try: from pymemcache.client.base import Client self.client = Client(url, timeout=3, connect_timeout=3) except ImportError: from memcache import Client self.client = Client([url], socket_timeout=3) self.ttl = ttl # Don't allow cross version caches # Pickle may have incompatibilities between versions self._key_prefix = '{}.{}.'.format(sys.version_info.major, sys.version_info.minor) def get(self, key): """Get the value belonging to a key. An error will be raised if the cache is expired or doesn't exist. """ value = self.client.get(self._key_prefix + key) if value is None: raise exceptions.CacheNotFound(key) return _decode(value) def put(self, key, value, ttl=None): """Write a new value to the cache. This will overwrite any old cache with the same key. """ if ttl is None: ttl = self.ttl self.client.set(self._key_prefix + key, _encode(value), ttl or 0, noreply=True) def delete(self, key): """Delete an item of cache if it exists.""" return self.client.delete(self._key_prefix + key, noreply=False)
def get_memcached(k): '''获取memcached数据 ''' try: client = Client(('172.20.10.7', 11211)) return json.load(client.get(k)) except Exception as e: print(e) return False
class MemcacheTool(object): def __init__(self, host, port): self.client = Client((host, port)) def set(self, key, value): self.client.set(key, value) def get(self, key): return self.client.get(key)
def test_memcached(mc_host, mc_port): # test that memcache is working from pymemcache.client.base import Client mc_client = Client((mc_host, mc_port)) my_id = str(uuid4()) my_val = str(uuid4()) mc_client.set(my_id, my_val, 1000) if mc_client.get(my_id).decode('utf8') != my_val: raise RuntimeError('Unable to communicate with Memcached')
class MemClient(): def __init__(self): self.client = Client(('localhost', 11211)) def get(self, key): value = self.client.get(key) return json.loads(value) def set(self, key, value): self.client.set(key, json.dumps(value), expire=18000)
class MyLib: def __init__(self): MEMCACHED_SERVER = os.getenv('MEMCACHED_SERVER', '127.0.0.1') MEMCACHED_PORT = os.getenv('MEMCACHED_PORT', 11211) #self.client = Client(('127.0.0.1', 11211)) self.client = Client((MEMCACHED_SERVER, MEMCACHED_PORT)) self.chunk_size = 1000000.0 def get_counts(self, file_size): return int(math.ceil(file_size / self.chunk_size)) def set_file(self, name, obj_file): file_size = len(obj_file.read()) if self.client.get("%s%s%s" % (name, "_", str(0))) is not None: return "File already exists." if file_size <= 50000000: obj_file.seek(0) counts = self.get_counts(file_size) self.client.set(name, counts) for i in range(counts): data = obj_file.read(int(self.chunk_size)) self.client.set("%s%s%s" % (name, "_", str(i)), data) return "Done." else: return "Oversized, rejected." def get_file(self, name): if self.client.get("%s%s%s" % (name, "_", str(0))) is None: return "File doest not exists." try: counts = int(self.client.get(name)) obj_file = b'' for i in range(counts): data = self.client.get("%s%s%s" % (name, "_", str(i))) obj_file = obj_file + data file = open(name, 'wb') file.write(obj_file) file.close() return name except: return None
def delMemcache(self, configkey, key, **kwargs): try: clients = self.getMemcacheConfig(configkey) for client_conf in clients: client = Client(client_conf) result = client.get(key=key) if result is not None: result = client.delete(key=key) logging.info("删除memcache的key={0},操作结果={1}".format(key, result)) except Exception as e: logging.error(e)
def fib_handler(num): client = Client((MEMCACHED_HOST, '11211'), serializer=json_serializer, deserializer=json_deserializer) fib_result = client.get(str(num)) if fib_result is None: result = fib(num) client.set(str(num), result) else: result = fib_result print('fib_result=', fib_result) return str(result)
def fib_handler(k): client = Client((MEMCACHED_HOST, '11211') , serializer=json_serializer , deserializer=json_deserializer) # клиент кэширования fib_result = client.get(str(k)) # число фибоначчи из кэша if fib_result is None: result = fib(k) client.set(str(k), result) # кэшируем число фибоначчи else: result = fib_result print('fib_result=', fib_result) return str(result)
def get_test(request): client = Client(('/opt/www/memcached/memcached.sock'), serializer=json_serializer, deserializer=json_deserializer) result = client.get('INDEX_ONLINE') if result is None: result = {'status': 'NotFound'} print(result) return JsonResponse(result)
class MemcachedCache(object): """ a memcached cache which stores the states as pickled objects """ def __init__(self, master): self.is_master = False self.wait_on_insert = master.wait_on_insert self.memc_params = master.memc_params self.val_ttl = master.val_ttl def __del__(self): self.teardown() def setup(self): self.client = Client((self.memc_params['host'], self.memc_params['port']), default_noreply=not self.wait_on_insert) return self def teardown(self): return self def fetch(self, routingkey, key): cache_key = str(routingkey) + '_' + str(key) obj_pickle = self.client.get(cache_key) if obj_pickle is None: return None try: obj = pickle.loads(obj_pickle) except: # logger.debug("Error decoding pickle") obj = None return obj def insert(self, routingkey, key, obj): cache_key = str(routingkey) + '_' + key obj_pickle = pickle.dumps(obj) if self.wait_on_insert: self.client.set(key=cache_key, value=obj_pickle, expire=self.val_ttl) else: threading.Thread(target=self.client.set, kwargs={'key': cache_key, 'value': obj_pickle, 'expire': self.val_ttl}, daemon=False).start() def get_client(self): return self.__copy__() def __copy__(self): if not self.is_master: raise Exception('not allowed; already a slave') return MemcachedCache(self) def copy(self): return self.__copy__()
def set_api_throttle(): client = Client((CACHE_CLIENT_ENDPOINT, CACHE_PORT)) result = client.get('daily_requests') if (result): result = int(result) + 1 else: result = 1 client.set('daily_requests',result, expire=CACHE_EXPIRE) return result
def get_api_throttle(): is_throttled = 0 #memecached client = Client((CACHE_CLIENT_ENDPOINT, CACHE_PORT)) result = client.get('daily_requests') if (result): result = int(result) if (result >= API_LIMIT): is_throttled = 1 return is_throttled
class MemcachedCache(CachualCache): """A cache using `Memcached <https://memcached.org/>`_ as the backing cache. The same caveats apply to keys and values as for Redis - you should only try to store strings (using the packing/unpacking functions). See the documentation on Keys and Values here: :class:`pymemcache.client.base.Client`. :type host: string :param host: The Memcached host to use for the cache. :type port: integer :param port: The port to use for the Memcached server. :type kwargs: dict :param kwargs: Any additional args to pass to the :class:`CachualCache` constructor. """ def __init__(self, host='localhost', port=11211, **kwargs): super(MemcachedCache, self).__init__(**kwargs) self.client = MemcachedClient((host, port)) def get(self, key): """Get a value from the cache using the given key. :type key: string :param key: The cache key to get the value for. :returns: The value for the cache key, or None in the case of cache miss. """ return self.client.get(key) def put(self, key, value, ttl=None): """Put a value into the cache at the given key. For constraints on keys and values, see :class:`pymemcache.client.base.Client`. :type key: string :param key: The cache key to use for the value. :param value: The value to store in the cache. :type ttl: integer :param ttl: The time-to-live for key in seconds, after which it will expire. """ if ttl is None: ttl = 0 self.client.set(key, value, expire=ttl)
def test_serialization_deserialization(host, port, socket_module): def _ser(key, value): return json.dumps(value).encode('ascii'), 1 def _des(key, value, flags): if flags == 1: return json.loads(value.decode('ascii')) return value client = Client((host, port), serializer=_ser, deserializer=_des, socket_module=socket_module) client.flush_all() value = {'a': 'b', 'c': ['d']} client.set(b'key', value) result = client.get(b'key') assert result == value
# check current input source no if handleCommand("47"): time.sleep(4) m, n = getscreenimage() return m, n #------------------------ #logging.basicConfig( # format='# %(levelname)s: %(message)s', # level=logging.DEBUG, #) client = Client(('127.0.0.1', 11211)) try: result = client.get('appletv') if result: sys.exit("time is not passed") else: print ("appletv is on") # get current channel and change input to HDMI2 # tv should be on m, n = getCHandHDMI2() if m != n: changeinout(m, n) else: handleCommand("20") client.set('appletv', 'true', 60) except Exception, e: print e.__doc__
import sys import StringIO from PIL import Image, ImageOps import images2gif from pymemcache.client.base import Client # http://stackoverflow.com/questions/19149643/error-in-images2gif-py-with-globalpalette # change palettes.append( getheader(im)[1] ) to palettes.append(im.palette.getdata()[1]) in images2gif.py client = Client(('127.0.0.1', 11211)) images = [] size = 256, 192 timespace = 10 try: result = client.get('curno') if result : curno = int(result) for i in range((curno - timespace), curno): data = client.get(str(i)) if data: im = Image.open(StringIO.StringIO(data)) imresized = ImageOps.fit(im, size, Image.ANTIALIAS) images.append(imresized) filename = "my_gif.GIF" images2gif.writeGif(filename, images, duration=0.2) except Exception, e: print e.__doc__ print e.message
def gao_memcache(): client = Client((app.config['MEMCACHE_HOST'], app.config['MEMCACHE_PORT'])) client.set('some_key', 'some_value', expire=10) result = client.get('some_key') print result
class TwitterLimits(object): """ Checks against either the twitter API or memcached twitter API results for twitter API requests limits """ def __init__(self, timeline_rate_reserve=5, multi_proc_logger=None): """ The reserve arguments are how many requests to hold back on to leave some form of buffer in place with regards to API limits """ if multi_proc_logger: self.logger = multi_proc_logger else: self.logger = utils.MultiProcessCheckingLogger(module_logger) self.memcacheClient = MemCacheClient( (interns_settings.memcache_host, interns_settings.memcache_port) ) self.timeline_rate_reserve = timeline_rate_reserve self.tl_total_reqs = interns_settings.twitter_timeline_requests self.tl_reqs_left = interns_settings.twitter_timeline_req_left self.tl_reqs_reset_time = interns_settings.twitter_timeline_reset_time self.update_limits() def update_limits(self): """ Update the limits associated with the twitter api First attempts to check memcache and then checks directly with the twitter API """ self.logger.debug(__name__, 'Updating twitter limits') cache_total_tl_reqs = self.memcacheClient.get('timeline_limit') if cache_total_tl_reqs: self.tl_total_reqs = int(cache_total_tl_reqs) cache_tl_reqs_left = self.memcacheClient.get('timeline_remaining') if cache_tl_reqs_left: self.tl_reqs_left = int(cache_tl_reqs_left) cache_tl_reqs_reset_time = self.memcacheClient.get( 'timeline_reset' ) if cache_tl_reqs_reset_time: self.tl_reqs_reset_time = int(cache_tl_reqs_reset_time) utc_now = datetime.utcnow() utc_secs = (utc_now - epoch_time).total_seconds() secs_until_reset = self.tl_reqs_reset_time - utc_secs if secs_until_reset <= 0: # Force getting rates from twitter self.tl_reqs_reset_time = None update_values_valid = ( self.tl_total_reqs and (self.tl_reqs_left is not None) and self.tl_reqs_reset_time ) if not update_values_valid: self.logger.debug(__name__, 'Making twitter API limits request') update_vals = ( twitterClient.get_user_timeline_rate_limit() ) self.tl_total_reqs = update_vals.limit self.tl_reqs_left = update_vals.remaining self.tl_reqs_reset_time = update_vals.reset else: self.logger.debug( __name__, 'Using twitter API limits from memcache' ) self.logger.debug( __name__, 'Total twitter timeline requests allowed is: {0}'.format( self.tl_total_reqs ) ) self.logger.debug( __name__, 'Number of twitter timeline requests left is: {0}'.format( self.tl_reqs_left ) ) self.logger.debug( __name__, 'Twitter timeline request reset time is: {0}'.format( self.tl_reqs_reset_time ) ) def get_sleep_between_jobs(self): """ Calculate the sleep time between jobs as to not run into the twitter API limits """ self.update_limits() self.logger.debug(__name__, 'Updating sleep time between jobs') utc_now = datetime.utcnow() utc_secs = (utc_now - epoch_time).total_seconds() self.logger.debug(__name__, 'UTC in seconds {0}'.format(utc_secs)) secs_until_reset = self.tl_reqs_reset_time - utc_secs self.logger.debug( __name__, 'Seconds until API reset {0}'.format(secs_until_reset) ) buffered_tl_reqs_left = ( self.tl_reqs_left - self.timeline_rate_reserve ) self.logger.debug( __name__, 'Buffered timeline requests left {0}'.format( buffered_tl_reqs_left ) ) if buffered_tl_reqs_left <= 0: sleep_time = secs_until_reset else: sleep_time = secs_until_reset / buffered_tl_reqs_left self.logger.debug(__name__, 'Sleep time {0}'.format(sleep_time)) return sleep_time