def lightning_order_with_redlock() -> None: """ Lightning order with Redlock algorithm. :return: None """ r = redis.Redis() dlm = Redlock([{ 'host': 'localhost', 'port': 6379, 'db': 0 }, ]) # Stands for "distributed lock manager" lock = None try: # Try to acquire the lock lock = dlm.lock(LOCK_KEY, 30000) # If not acquiring the lock, block here # Business codes remaining = int(r.get('stock')) if remaining > 0: r.set('stock', str(remaining - 1)) print(f'Deducted stock, {remaining - 1} remaining') else: print('Failed to deduct stock') except MultipleRedlockException as e: print(e) finally: # Release the lock dlm.unlock(lock)
class RedisLock(object): def __init__(self, redis_host='127.0.0.1', redis_port=6379): self._lock_manager = Redlock([{ "host": redis_host, "port": redis_port, "db": 0, }]) @contextmanager def lock(self, lock_id, timeout=10000): ''' Context manager to acquire global distributed lock across all applications using the same redis. Uses implementation of redis distlock @see https://redis.io/topics/distlock @lock_id string uniq id of the lock @timeout integer timeout after which the lock will be released (in milliseconds) ''' lock = self._lock_manager.lock(lock_id, timeout) if lock is False: raise LockFailed() try: yield lock finally: self._lock_manager.unlock(lock)
class RedisClient(object): def __init__(self, host, port=6379, db=0): self.redis = redis.StrictRedis(host=host, port=port, db=db) self.redlock = Redlock([self.redis]) def __getattr__(self, name): method = getattr(self.redis, name) return method def acquireDLock(self, resource, ttl=300): """ 获取分布式锁。 ttl: 锁的过期时间,单位是秒,默认300秒。 """ resource = 'lock_' + resource try: return self.redlock.lock(resource, ttl * 1000) except MultipleRedlockException as e: logging.error('Error acquiring dlock: %s', e) return False def releaseDLock(self, lock): """ 释放分布式锁 """ try: self.redlock.unlock(lock) return True except MultipleRedlockException as e: logging.error('Error releasing dlock: %s', e) return False
def dlock(key, ttl, **kwargs): """ 分布式锁 :param key: 分布式锁ID :param ttl: 分布式锁生存时间 :param kwargs: 可选参数字典 :return: None """ resource_servers = [{ 'host': REDIS_HOST, 'port': REDIS_PORT, 'db': REDIS_DB, 'password': REDIS_PASSWORD }] dl = Redlock(resource_servers) # 获取锁 lock = dl.lock(key, ttl) # if ret is False: # detail = u'acquire lock[%s] error' % key # raise AcquireLockError(detail) yield lock # 释放锁 if isinstance(lock, Lock): dl.unlock(lock)
class RedisStore(AbstractStore): """Redis-based backend for deployments with replicas > 1""" def __init__(self, url: str): logger.info("Connecting to Redis on {}..".format(url)) self._redis = redis.StrictRedis.from_url(url) self._redlock = Redlock([url]) def set(self, key, value): self._redis.set(key, json.dumps(value, separators=(",", ":"))) def get(self, key): value = self._redis.get(key) if value: return json.loads(value.decode("utf-8")) def acquire_lock(self): return self._redlock.lock("update", 10000) def release_lock(self, lock): self._redlock.unlock(lock) def publish(self, event_type, event_data): self._redis.publish( "default", "{}:{}".format(event_type, json.dumps(event_data, separators=(",", ":"))), ) def listen(self): p = self._redis.pubsub() p.subscribe("default") for message in p.listen(): if message["type"] == "message": event_type, data = message["data"].decode("utf-8").split( ":", 1) yield (event_type, json.loads(data)) def create_screen_token(self): """Generate a new screen token and store it in Redis""" data = generate_token_data() token = data["token"] self._redis.set("screen-tokens:{}".format(token), json.dumps(data)) return token def redeem_screen_token(self, token: str, remote_addr: str): """Validate the given token and bind it to the IP""" redis_key = "screen-tokens:{}".format(token) data = self._redis.get(redis_key) if not data: raise ValueError("Invalid token") data = json.loads(data.decode("utf-8")) data = check_token(token, remote_addr, data) self._redis.set(redis_key, json.dumps(data))
class RedisStore(AbstractStore): '''Redis-based backend for deployments with replicas > 1''' def __init__(self, url: str): logger.info('Connecting to Redis on {}..'.format(url)) self._redis = redis.StrictRedis.from_url(url) self._redlock = Redlock([url]) def set(self, key, value): self._redis.set(key, json.dumps(value, separators=(',', ':'))) def get(self, key): value = self._redis.get(key) if value: return json.loads(value.decode('utf-8')) def acquire_lock(self): return self._redlock.lock('update', 10000) def release_lock(self, lock): self._redlock.unlock(lock) def publish(self, event_type, event_data): self._redis.publish( 'default', '{}:{}'.format(event_type, json.dumps(event_data, separators=(',', ':')))) def listen(self): p = self._redis.pubsub() p.subscribe('default') for message in p.listen(): if message['type'] == 'message': event_type, data = message['data'].decode('utf-8').split( ':', 1) yield (event_type, json.loads(data)) def create_screen_token(self): '''Generate a new screen token and store it in Redis''' data = generate_token_data() token = data['token'] self._redis.set('screen-tokens:{}'.format(token), json.dumps(data)) return token def redeem_screen_token(self, token: str, remote_addr: str): '''Validate the given token and bind it to the IP''' redis_key = 'screen-tokens:{}'.format(token) data = self._redis.get(redis_key) if not data: raise ValueError('Invalid token') data = json.loads(data.decode('utf-8')) data = check_token(token, remote_addr, data) self._redis.set(redis_key, json.dumps(data))
class TestRedlock(APITestCase): def setUp(self): self.redlock = Redlock([{"host": "localhost"}]) def test_lock(self): lock = self.redlock.lock("pants", 100) print(lock) self.assertEqual(lock.resource, "pants") self.redlock.unlock(lock) lock = self.redlock.lock("pants", 10) print(lock) self.redlock.unlock(lock)
class RedisStore(AbstractStore): '''Redis-based backend for deployments with replicas > 1''' def __init__(self, url: str): logger.info('Connecting to Redis on {}..'.format(url)) self._redis = redis.StrictRedis.from_url(url) self._redlock = Redlock([url]) def set(self, key, value): self._redis.set(key, json.dumps(value, separators=(',', ':'))) def get(self, key): value = self._redis.get(key) if value: return json.loads(value.decode('utf-8')) def acquire_lock(self): return self._redlock.lock('update', 10000) def release_lock(self, lock): self._redlock.unlock(lock) def publish(self, event_type, event_data): self._redis.publish('default', '{}:{}'.format(event_type, json.dumps(event_data, separators=(',', ':')))) def listen(self): p = self._redis.pubsub() p.subscribe('default') for message in p.listen(): if message['type'] == 'message': event_type, data = message['data'].decode('utf-8').split(':', 1) yield (event_type, json.loads(data)) def create_screen_token(self): '''Generate a new screen token and store it in Redis''' data = generate_token_data() token = data['token'] self._redis.set('screen-tokens:{}'.format(token), json.dumps(data)) return token def redeem_screen_token(self, token: str, remote_addr: str): '''Validate the given token and bind it to the IP''' redis_key = 'screen-tokens:{}'.format(token) data = self._redis.get(redis_key) if not data: raise ValueError('Invalid token') data = json.loads(data.decode('utf-8')) data = check_token(token, remote_addr, data) self._redis.set(redis_key, json.dumps(data))
class TestRedlock(unittest.TestCase): def setUp(self): self.redlock = Redlock([{"host": "localhost"}]) def test_lock(self): lock = self.redlock.lock("pants", 100) self.assertEqual(lock.resource, "pants") self.redlock.unlock(lock) lock = self.redlock.lock("pants", 10) self.redlock.unlock(lock) def test_blocked(self): lock = self.redlock.lock("pants", 1000) bad = self.redlock.lock("pants", 10) self.assertFalse(bad) self.redlock.unlock(lock) def test_bad_connection_info(self): with self.assertRaises(Warning): Redlock([{"cat": "hog"}]) def test_py3_compatible_encoding(self): lock = self.redlock.lock("pants", 1000) key = self.redlock.servers[0].get("pants") self.assertEquals(lock.key, key)
def id(self): deviceIP = self.client.host self.session_id = self.get_redis_session(deviceIP) if not self.check_session_invalid(self.session_id, deviceIP): local_ip = self.get_local_ip() redisobj = RedisConfig() redis_server = redisobj.get_redis_servers(True) redis_conf_ls = [{ "host": s["host"], "port": s["port"], "db": s["dbno"] } for s in redis_server] lock_mgmt = Redlock(redis_conf_ls) device_lock = lock_mgmt.lock( local_ip + "_" + deviceIP + '_' + self.username + '_device_lock_calabash', 30 * 1000) tmp_count = 0 while isinstance(device_lock, bool) and not device_lock and tmp_count < 1000: tmp_count += 1 time.sleep(0.5) self.session_id = self.get_redis_session(deviceIP) if self.check_session_invalid(self.session_id, deviceIP): return self.session_id device_lock = lock_mgmt.lock( local_ip + "_" + deviceIP + '_' + self.username + '_device_lock_calabash', 30 * 1000) self.session_id = self.get_redis_session(deviceIP) if self.check_session_invalid(self.session_id, deviceIP): return self.session_id self.authenticate(self.username, self.password) #set sessionid to redis self.set_session_to_redis(self.session_id, deviceIP) lock_mgmt.unlock(device_lock) return self.session_id
class SlaveSpider(Spider): name = 'slave_spider' default_retry_count = 3 default_retry_delay = 1 def __init__(self): self.dlm = Redlock([{"host": "42.96.132.158", "port": 6379, "db": 0, 'password': "******"}, ]) self.list = [] self.driver = PhantomJS() def start_requests(self): self.load_payload() while True: my_lock = self.dlm.lock('URLSLOCK', 1000) while isinstance(my_lock, bool): pass len = self.dlm.servers[0].llen('urls') while len > 0: url = self.dlm.servers[0].lpop('urls') self.dlm.unlock(my_lock) urls = self.get_urls(url=url) for u in urls: yield Request(url=u, callback=self.parse) log.msg('the slave spider is over!', log.WARNING) def parse(self, response): log.msg(response.url+' is over', log.INFO) def get_urls(self, url): for l in self.list: temp = url + l yield temp def load_payload(self): f = open('Xspider/payload.txt', 'r') for line in f.readlines(): line = line.strip('\n') self.list.append(line) f.close()
class TestRedlock(unittest.TestCase): def setUp(self): try: self.redlock = Redlock([{"host": "localhost"}]) self.dstlock = Redlock([{"host": "localhost", "port": 6379, "socket_timeout": 0.5}, {"host": "localhost", "port": 6380, "socket_timeout": 0.5}, {"host": "localhost", "port": 6381, "socket_timeout": 0.5}]) except Exception as e: pass ''' def test_distribute_locks(self): import time while True: if not self.dstlock.lock("fizz", 1000): print ("lock failed") else: print("lock success") time.sleep(1) ''' def test_lock(self): lock = self.redlock.lock("pants", 100) self.assertEqual(lock.resource, "pants") self.redlock.unlock(lock) lock = self.redlock.lock("pants", 10) self.redlock.unlock(lock) def test_blocked(self): lock = self.redlock.lock("pants", 1000) bad = self.redlock.lock("pants", 10) self.assertFalse(bad) self.redlock.unlock(lock) def test_bad_connection_info(self): with self.assertRaises(Warning): Redlock([{"cat": "hog"}]) def test_py3_compatible_encoding(self): lock = self.redlock.lock("pants", 1000) key = self.redlock.servers[0].get("pants") self.assertEqual(lock.key, key) def test_ttl_not_int_trigger_exception_value_error(self): with self.assertRaises(ValueError): self.redlock.lock("pants", 1000.0) def test_multiple_redlock_exception(self): ex1 = Exception("Redis connection error") ex2 = Exception("Redis command timed out") exc = MultipleRedlockException([ex1, ex2]) exc_str = str(exc) self.assertIn('connection error', exc_str) self.assertIn('command timed out', exc_str)
class RedisLocker(ILocker): def __init__(self, redis_con): self.dlm = Redlock([redis_con, ], retry_count=1) self._locks = {} def _lock(self, key, timeout): """ Trying to acquire lock If not success then sleep (timeout // 10) milliseconds and try again """ lock_key = f'lock/{key}' sleep_time = timeout / (10 * 1000) while True: lock = self.dlm.lock(lock_key, timeout) if lock: break time.sleep(sleep_time) self._locks[lock_key] = lock def _unlock(self, key): lock_key = f'lock/{key}' lock = self._locks[lock_key] self.dlm.unlock(lock)
class TestRedlock(unittest.TestCase): def setUp(self): self.redlock = Redlock([{"host": "localhost"}]) def test_lock(self): lock_name = "test_lock" lock = self.redlock.lock(lock_name, 100, TEST_LOCK_TTL_MILLIS) self.assertEqual(lock.resource, lock_name) self.redlock.unlock(lock) lock = self.redlock.lock(lock_name, 10, TEST_LOCK_TTL_MILLIS) self.redlock.unlock(lock) def test_blocked(self): lock_name = "test_blocked" lock = self.redlock.lock(lock_name, 1000, TEST_LOCK_TTL_MILLIS) with self.assertRaises(MultipleRedlockException): self.redlock.lock(lock_name, 10, TEST_LOCK_TTL_MILLIS) self.redlock.unlock(lock) def test_bad_connection_info(self): with self.assertRaises(Warning): Redlock([{"cat": "hog"}]) def test_py3_compatible_encoding(self): lock_name = "test_py3_compatible_encoding" lock = self.redlock.lock(lock_name, 1000, TEST_LOCK_TTL_MILLIS) key = self.redlock.servers[0].get(lock_name) self.assertEqual(lock.key, int(key)) def test_ttl_not_int_trigger_exception_value_error(self): lock_name = "ttl_not_int" with self.assertRaises(ValueError): self.redlock.lock(lock_name, 1000.0, 1000.0) def test_multiple_redlock_exception(self): ex1 = Exception("Redis connection error") ex2 = Exception("Redis command timed out") exc = MultipleRedlockException([ex1, ex2]) exc_str = str(exc) self.assertIn('connection error', exc_str) self.assertIn('command timed out', exc_str)
class TestRedlock(unittest.TestCase): def setUp(self): self.redlock = Redlock([{"host": "localhost"}]) def test_lock(self): lock = self.redlock.lock("pants", 100) self.assertEqual(lock.resource, "pants") self.redlock.unlock(lock) lock = self.redlock.lock("pants", 10) self.redlock.unlock(lock) def test_blocked(self): lock = self.redlock.lock("pants", 1000) bad = self.redlock.lock("pants", 10) self.assertFalse(bad) self.redlock.unlock(lock) def test_bad_connection_info(self): with self.assertRaises(Warning): Redlock([{"cat": "hog"}]) def test_py3_compatible_encoding(self): lock = self.redlock.lock("pants", 1000) key = self.redlock.servers[0].get("pants") self.assertEquals(lock.key, key) def test_ttl_not_int_trigger_exception_value_error(self): with self.assertRaises(ValueError): self.redlock.lock("pants", 1000.0) def test_multiple_redlock_exception(self): ex1 = Exception("Redis connection error") ex2 = Exception("Redis command timed out") exc = MultipleRedlockException([ex1, ex2]) exc_str = str(exc) self.assertIn('connection error', exc_str) self.assertIn('command timed out', exc_str)
traces_path = sys.argv[3] if not traces_path[-1] == '/': traces_path += "/" # To consume messages consumer = KafkaConsumer(topic, group_id='traces_cache', bootstrap_servers=[kafka_connection]) consumer.commit() dlm = Redlock([ { "host": "localhost", "port": 6379, "db": 0 }, ]) for message in consumer: key = message.key.decode('utf-8') lock = False while not lock: lock = dlm.lock(key, 5000) with open(traces_path + key, 'ab') as f: f.write(message.value) dlm.unlock(lock) consumer.task_done(message) consumer.commit()
# line = line.strip('\n') # url = url + line # list.append(url) # print list from redlock import Redlock from redlock import MultipleRedlockException import time dlm = Redlock([ { "host": "42.96.132.158", "port": 6379, "db": 0, 'password': "******" }, ]) try: dlm.servers[0].flushall() my_lock = dlm.lock('LOCK', 1000) while True: if isinstance(my_lock, bool): print 'wait' time.sleep(0.5) else: print 'dosomething' dlm.servers[0].lpush('urls', time.time()) dlm.unlock(my_lock) time.sleep(0.5) except MultipleRedlockException, e: raise e
class ProbitScheduler(Scheduler): def __init__(self, redis_connection=None, locker=None, *args, **kwargs): self.__redis_connection = redis_connection if self.__redis_connection is None: self.__redis_connection = StrictRedis.from_url( current_app.conf.CELERY_REDIS_SCHEDULER_URL) self._schedule = EntryProxy(self.__redis_connection) self._locker = locker if self._locker is None: self._locker = Redlock( [current_app.conf.CELERY_REDIS_SCHEDULER_URL]) super(ProbitScheduler, self).__init__(*args, **kwargs) def setup_schedule(self): self.install_default_entries(self._schedule) self._merge(self.app.conf.CELERYBEAT_SCHEDULE) def get_schedule(self): return self._schedule schedule = property( get_schedule) # This isn't inherited anymore? Do we want to do this? def sync(self): # Reload the schedule from the collection self._schedule = EntryProxy(self.__redis_connection) # I'm not sure what reserve() is intended to do, but it does not do what we # need it to do, so we define a _lock() method as well. def maybe_due(self, entry, publisher=None): is_due, next_time_to_run = entry.is_due() if not is_due: return next_time_to_run lock = self._lock(entry.name) if not lock: return next_time_to_run try: # Now that we have the lock, double-check the timestamps on the # entry before executing it. entry = self._schedule.sync(entry.name) if entry is None: return next_time_to_run is_due, next_time_to_run = entry.is_due() if not is_due: return next_time_to_run return Scheduler.maybe_due(self, entry, publisher) finally: self._unlock(lock) def _lock(self, name): return self._locker.lock(name, 1000) def _unlock(self, lock): self._locker.unlock(lock) def _merge(self, schedule): """schedule_keys = self.__redis_connection.hgetall(ENTRY_LIST_KEY).keys() if len(schedule_keys) > 0: self.__redis_connection.hdel(ENTRY_LIST_KEY, *schedule_keys)""" for name, entry_dict in list(schedule.items()): entry = ScheduleEntry(name, **entry_dict) if name not in self._schedule: self._schedule[name] = entry else: # _lock() the existing entry so that these values aren't changed # while we're merging them. lock = self._lock(name) if lock: try: existing = self._schedule.sync(name) if existing: entry.last_run_at = existing.last_run_at entry.total_run_count = existing.total_run_count self._schedule[name] = entry finally: self._unlock(lock)
gameplays = db.gameplays.find({'modified': True}) for gameplay in gameplays: try: gameplay_file = '%s' % gameplay['_id'] lock = False while not lock: lock = dlm.lock(gameplay_file, 5000) try: shutil.move(traces_path + gameplay_file, cache_path + gameplay_file) except FileNotFoundError as e: print(e) continue finally: dlm.unlock(lock) db.gameplays.update_one({'_id': gameplay['_id'], 'lastAccessed': gameplay['lastAccessed']}, {'$set': {'modified': False}}) with open(cache_path + gameplay_file, 'rb') as src: with open(storage_path + gameplay_file, 'ab') as dst: dst.write(src.read()) with open(cache_path + gameplay_file, 'r') as f: traces = f.read().split('\n') collection_name = 'gameplays_results_' + str(gameplay['gameId']) if collection_name not in db.collection_names(): db.create_collection(collection_name) gameplays_results = db[collection_name] result = gameplays_results.find_one({'gameplayId': gameplay['_id']})
class ConsumerQueue(object): def __init__(self, redis_host=None, redis_port=None, redis_db=None, producer_queue_id=None, consumer_queue_id=None): assert redis_host assert redis_port assert redis_db assert producer_queue_id assert consumer_queue_id pool = redis.ConnectionPool(host=redis_host, port=redis_port, db=redis_db) self._broker = redis.Redis(connection_pool=pool) self._consumer_queue_id = producer_queue_id self._consumer_queue_id = consumer_queue_id self._producer_queue_id = producer_queue_id self._lock_manager = Redlock([{ 'host': redis_host, 'port': redis_port, 'db': redis_db}, ]) def dequeue_task(self): """ Pulls the descriptor for a task that is ready for processing returns a descriptor or None if no tasks are present """ # do we still have work items in this worker's queue descriptor = self._broker.lindex(self._consumer_queue_id, 0) if not descriptor: # check the global work queue for work descriptor = self._broker.rpoplpush(self._producer_queue_id, self._consumer_queue_id) return descriptor def retire_task(self, descriptor=None): """ Mark a task as having been successfully dispatched """ assert descriptor popped_id = self._broker.lpop(self._consumer_queue_id) assert popped_id == descriptor return popped_id def lock_work_item(self, descriptor=None, lock_timeout=None): """ Locks a task to prevent other consumers from double processing """ assert descriptor assert lock_timeout lock = self._lock_manager.lock(descriptor, lock_timeout) return lock def unlock_task(self, lock=None): assert lock self._lock_manager.unlock(lock)
class CacheManager(object): def __init__(self, redis_engine, lock_ttl=1000): self.redis = redis_engine self.redlock = Redlock([redis_engine]) self.lock_ttl = lock_ttl @staticmethod def get_lock_key(key): return "%s:lock" % key def lock(self, key, nowait=False): lock_key = self.get_lock_key(key) l = False while not l: l = self.redlock.lock(lock_key, self.lock_ttl) if l or nowait: return l time.sleep(0.3) def unlock(self, lock): self.redlock.unlock(lock) def get_item(self, key, recover=None, ttl=None, arguments=([], {}), **kw): item_key = self.redis.get(key) value = self.redis.get(item_key) if item_key else None # return None if cannot recover the value if value is None and recover is None: return None # Trying to recover the value if value is None: return self.set_item(key, lambda: recover(*arguments[0], **arguments[1]), ttl=ttl) else: return deserialize(value) def set_item(self, key, value, ttl=None): old_item_key = self.redis.get(key) if old_item_key is None: # this is a new item # locking it to prevent concurrency violation lock = self.lock(key, nowait=True) if not lock: # it seems this item is loading in another thread # so, waiting for that: # wait & make sure the object is reloaded, the release the lock self.unlock(self.lock(key)) return self.get_item(key) # check if item loaded v = self.get_item(key, None) if v: self.unlock(lock) return v else: lock = None try: guid = uuid.uuid1() item_key = "%s:%s" % (key, guid) value = value() if callable(value) else value self.redis.set(item_key, serialize(value), ex=ttl) self.redis.set(key, item_key, ex=ttl) if old_item_key: self.redis.delete(old_item_key) return value finally: if lock: self.unlock(lock) def hget_item(self, key, dict_key, recover=None, ttl=None, arguments=([], {}), **kw): item_key = self.redis.get(key) value = self.redis.hget(item_key, dict_key) if item_key else None # return None if cannot recover the value if value is None and recover is None: return None # Trying to recover the value if value is None: return self.hset_item(key, dict_key, lambda: recover(*arguments[0], **arguments[1]), ttl=ttl) else: return deserialize(value) def hset_item(self, key, dict_key, value, ttl=None): old_item_key = self.redis.get(key) if old_item_key is None: # this is a new item # locking it to prevent concurrency violation lock = self.lock(key, nowait=True) if not lock: # it seems this item is loading in another thread # so, waiting for that: # wait & make sure the object is reloaded, the release the lock self.unlock(self.lock(key)) return self.hget_item(key, dict_key) # check if item loaded v = self.hget_item(key, dict_key, None) if v: self.unlock(lock) return v else: lock = None try: guid = uuid.uuid1() item_key = "%s:%s" % (key, guid) value = value() if callable(value) else value self.redis.hset(item_key, dict_key, serialize(value)) self.redis.expire(item_key, time=ttl) self.redis.set(key, item_key, ex=ttl) if old_item_key: self.redis.delete(old_item_key) return value finally: if lock: self.unlock(lock) def del_item(self, *keys): item_keys_to_remove = [] for key in keys: item_key = self.redis.get(key) if item_key is not None: item_keys_to_remove.append(item_key) self.redis.delete(*item_keys_to_remove) self.redis.delete(*keys) def get_list(self, key, recover=None, ttl=None, arguments=([], {}), key_extractor=None): # First, get the keys list value = self.redis.get(key) if value is None and recover is None: return None, None # Trying to recover the value if value is None: return self.set_list( key, lambda: recover(*arguments[0], **arguments[1]), ttl=ttl, key_extractor=key_extractor ) else: metadata, item_keys = deserialize(value) return metadata, [self.get_item(item_key) for item_key in item_keys] def set_list(self, key, value, ttl=None, key_extractor=None): # locking it to prevent concurrency violation lock = self.lock(key) # check if item loaded v = self.get_list(key) if v[1]: self.unlock(lock) return v # packed: (value_metadata, value) if key_extractor is None: def key_extractor(x): return x["id"] try: value_metadata, value = value() if callable(value) else value item_keys = [] assert isinstance(value, collections.Iterable), "Value must be iterable" for item_value in value: item_key = create_cache_key(key.split(":")[0], key_extractor(item_value)) item_keys.append(item_key) self.set_item(item_key, item_value, ttl=ttl) self.redis.set(key, serialize((value_metadata, item_keys)), ex=ttl) return value_metadata, value finally: if lock: self.unlock(lock) def invalidate_item(self, key): item_key = self.redis.get(key) self.redis.delete(key) if item_key: self.redis.delete(item_key) def invalidate_list(self, key): self.redis.delete(key) def decorate(self, namespace, list_=False, ttl=None, key_extractor=None): def decorator(func): def wrapper(*args, **kwargs): cache_key = create_cache_key(namespace, kwargs) cache_params = dict(ttl=ttl, recover=func, arguments=(args, kwargs)) if list_: cache_method = self.get_list cache_params["key_extractor"] = key_extractor else: cache_method = self.get_item return cache_method(cache_key, **cache_params) return wrapper return decorator
# host = '42.96.132.158' # psd = 'wjdh84928399' # r = redis.Redis(host=host, password=psd, db=0, port=6379, socket_timeout=10) # url = r.lpop('urls') # f = open('Xspider/payload.txt', 'r') # list = [] # for line in f.readlines(): # line = line.strip('\n') # url = url + line # list.append(url) # print list from redlock import Redlock from redlock import MultipleRedlockException import time dlm = Redlock([{"host": "42.96.132.158", "port": 6379, "db": 0, 'password': "******"}, ]) try: dlm.servers[0].flushall() my_lock = dlm.lock('LOCK', 1000) while True: if isinstance(my_lock, bool): print 'wait' time.sleep(0.5) else: print 'dosomething' dlm.servers[0].lpush('urls', time.time()) dlm.unlock(my_lock) time.sleep(0.5) except MultipleRedlockException, e: raise e
class RecordLockManager(object): def __init__(self, conf, prefix=None, ttl_sec=60, backlog_path=None): self.backlog_lock = Lock() self.conf = conf self.redlock = Redlock([{"host": conf['backend']['host'], "port": int(conf['backend']['port']), "db": 13, "password": conf['backend']['password']}, ], retry_count=100000, retry_delay=0.01) if prefix is not None: self.prefix = prefix + "-" self.ttl = ttl_sec * 1000 self.backlog_filename_pat = re.compile(self.prefix + '([\w\d\.\-\\/]+?)\-backlog\.([\w\d\-]+)') self.backlog = {} self.backlog_path = backlog_path # Should be abs path to the dir containing backlog pickles self.is_dumping = False if not os.path.exists(self.backlog_path): os.makedirs(self.backlog_path) self.reload() def parse_backlog_filename(self, name): match = self.backlog_filename_pat.search(name) if match is not None: return match.group(1), match.group(2) else: return None, None def reload(self): for i in os.listdir(self.backlog_path): if os.path.isfile(os.path.join(self.backlog_path, i)): name, postfix = self.parse_backlog_filename(i) if name is not None: with open(self.make_real_path(i), 'rb') as ifile: a = pickle.load(ifile) (data, saved_context) = a task = self.make_task(name, data, postfix=postfix) task.saved_context = saved_context task.is_backlog = True self.backlog[task.pickle_name] = task def make_real_path(self, pickle_name): if self.backlog_path is not None: return os.path.join(self.backlog_path, pickle_name) else: return pickle_name # TODO: implement exponential back off def choose_backlog_and_lock_it(self, ttl_sec=None, retry=5): with self.backlog_lock: count = 0 while count < retry: if len(self.backlog) < 1: return None backlog_task = self.backlog[random.choice(self.backlog.keys())] if backlog_task.lock is not None: # this task is already locked, maybe by other thread continue count+=1 self.try_lock(backlog_task, ttl_sec) if not backlog_task.lock: return None else: return backlog_task return None def try_lock(self, task, ttl_sec=None, block=False): if ttl_sec is not None: ttl = ttl_sec * 1000 else: ttl = self.ttl lock = self.redlock.lock(task.lock_name, ttl) while block and not lock: lock = self.redlock.lock(task.lock_name, ttl) if not lock: task.lock = None else: task.lock = lock def store_backlog(self, task, saved_context): print "Store backlog " + task.lock_name self.is_dumping = True task.saved_context = saved_context with open(self.make_real_path(task.pickle_name), "wb") as ofile: pickle.dump((task.data, saved_context), ofile) self.is_dumping = False with self.backlog_lock: # Must ensure task holds no lock before store it as backlog # otherwise, it may never scheduled again if task.lock is not None: self.unlock(task.lock) task.lock = None task.is_backlog = True self.backlog[task.pickle_name] = task def make_task(self, name, data, postfix=None): return Task(name, self.prefix, data, is_backlog=False, postfix=postfix) def unlock(self, lock): self.redlock.unlock(lock) def commit_task(self, task): if task.is_backlog: with self.backlog_lock: if task.pickle_name in self.backlog: del self.backlog[task.pickle_name] real_path = self.make_real_path(task.pickle_name) if os.path.exists(real_path): os.remove(real_path) if task.lock is not None: self.unlock(task.lock) task.lock = None
class ProbitScheduler(Scheduler): def __init__(self, redis_connection=None, locker=None, *args, **kwargs): self.__redis_connection = redis_connection if self.__redis_connection is None: self.__redis_connection = StrictRedis.from_url(current_app.conf.CELERY_REDIS_SCHEDULER_URL) self._schedule = EntryProxy(self.__redis_connection) self._locker = locker if self._locker is None: self._locker = Redlock([current_app.conf.CELERY_REDIS_SCHEDULER_URL]) super(ProbitScheduler, self).__init__(*args, **kwargs) def setup_schedule(self): self.install_default_entries(self._schedule) self._merge(self.app.conf.CELERYBEAT_SCHEDULE) def get_schedule(self): return self._schedule schedule = property(get_schedule) # This isn't inherited anymore? Do we want to do this? def sync(self): # Reload the schedule from the collection self._schedule = EntryProxy(self.__redis_connection) # I'm not sure what reserve() is intended to do, but it does not do what we # need it to do, so we define a _lock() method as well. def maybe_due(self, entry, publisher=None): is_due, next_time_to_run = entry.is_due() if not is_due: return next_time_to_run lock = self._lock(entry.name) if not lock: return next_time_to_run try: # Now that we have the lock, double-check the timestamps on the # entry before executing it. entry = self._schedule.sync(entry.name) if entry is None: return next_time_to_run is_due, next_time_to_run = entry.is_due() if not is_due: return next_time_to_run return Scheduler.maybe_due(self, entry, publisher) finally: self._unlock(lock) def _lock(self, name): return self._locker.lock(name, 1000) def _unlock(self, lock): self._locker.unlock(lock) def _merge(self, schedule): """schedule_keys = self.__redis_connection.hgetall(ENTRY_LIST_KEY).keys() if len(schedule_keys) > 0: self.__redis_connection.hdel(ENTRY_LIST_KEY, *schedule_keys)""" for name, entry_dict in schedule.items(): entry = ScheduleEntry(name, **entry_dict) if name not in self._schedule: self._schedule[name] = entry else: # _lock() the existing entry so that these values aren't changed # while we're merging them. lock = self._lock(name) if lock: try: existing = self._schedule.sync(name) if existing: entry.last_run_at = existing.last_run_at entry.total_run_count = existing.total_run_count self._schedule[name] = entry finally: self._unlock(lock)