def aggregate_day_after_hour_aggregated(sender, left_boundary, right_boundary, **kwargs): aggregate_queryset = HourAggregate.objects.filter( timestamp__gte=left_boundary, timestamp__lt=right_boundary) aggregate_queryset.query.clear_ordering(force_empty=True) aggregate_queryset = aggregate_queryset.values( 'admin_id', 'instance_id', 'instance_name', 'source').annotate(value=Sum('value')) day_date = datetime(left_boundary.year, left_boundary.month, left_boundary.day, tzinfo=pytz.utc) with Lock(redis=redis, name='day_aggregation_{}'.format(day_date.isoformat())): for aggregate_dict in aggregate_queryset.iterator(): filter_data = { 'timestamp': day_date, 'instance_name': aggregate_dict.get('instance_name'), 'source': aggregate_dict.get('source'), 'admin_id': aggregate_dict.get('admin_id') } if not DayAggregate.objects.filter(**filter_data).update( value=F('value') + aggregate_dict.get('value')): DayAggregate.objects.create(timestamp=day_date, **aggregate_dict)
def get_hash_cache(name, key, ex=None, none_callback=None, *args, **kwargs): """ name : redis hash name key : redis 键名 ex : 超期时间 (单位:秒) none_callback : 返回None值,调用回调函数 args : 回调参数 kwags : 回调参数 """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(name), timeout=20): v = cli.hget(name, key) if v is None and none_callback: if DEBUG_CACHE: print "no redis cache hit. %s." % key v = none_callback(*args, **kwargs) try: nv = JSONDataEncoder.encode(v) except: nv = v cli.hset(name, key, nv) _set_expire_time(cli, name, ex) else: if DEBUG_CACHE: print "cache redis hit. %s." % key try: v = JSONDataEncoder.decode(v) except: pass return v
def get_hash_map_cache(name, keys, ex=None, none_callback=None, *args, **kwargs): """ name : name keys : key list ex : 超期时间 (单位:秒) none_callback : 返回None值,调用回调函数 args : 回调参数 kwags : 回调参数 """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(name), timeout=20): v = cli.hmget(name, keys) if v is None and none_callback: if DEBUG_CACHE: print "no redis cache hit. %s." % keys v = none_callback(*args, **kwargs) cli.hmset(name, v) _set_expire_time(cli, name, ex) if v and (isinstance(keys, str) or (isinstance(keys, list) and len(keys) == 1)): v = v[-1] return v
def finalize(self): """ Call before you generate a dataset This caches all the guids in the db at time of calling to a list creating a data structure which allows for fast access by the dataset object and fixing the length of the the dataset """ lockname = self.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'getting lock {lockname}') self.redis.set(self.key('finalized'), 'FINALIZED') if lk.owned(): lk.release() #logger.debug(f'released lock {lockname}') self.episodes = [] self.episode_len = [] for episode in range(self.num_episodes()): self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode()) self.episodes = sorted(self.episodes) for episode_id in self.episodes: self.episode_len.append(len(Episode(self, self.redis, episode_id))) self.episode_off = [] offset = 0 for l in self.episode_len: self.episode_off.append(offset) offset += l self.finalized = True
def refresh_redis_cache(name, ex): """ name : name ex : 超期时间 (单位:秒) """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(name), timeout=20): _set_expire_time(cli, name, ex)
def create_lock(cls, id, r_obj=None): import config from redis.lock import LuaLock as Lock redis_key = 'lock_%s_%s' % (cls.__name__, id) return Lock(r_obj or rds_tmp, redis_key, sleep=config.REDIS_LOCK_SLEEP, timeout=config.REDIS_LOCK_TIMEOUT)
def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs): self._unique_id = unique_id self._redis_server = redis_server or StrictRedis() refresh_lock = Lock(redis=self._redis_server, name='{0}_lock'.format(self._unique_id)) super(RedisManagedOAuth2Mixin, self).__init__(*args, refresh_lock=refresh_lock, **kwargs) if self._access_token is None: self._get_and_update_current_tokens()
def acquire(self, lox_name, lock_id, expires_seconds=None): key = self.key(lox_name, lock_id) lock = Lock(self.connection, key, timeout=expires_seconds) # retry logic is handled in core.lock, so no blocking here if lock.acquire(blocking=False): return BackendLock(key, lox_name, lock_id, provider_lock=lock) else: raise LockInUseException( "Lock {} has been acquired previously, possibly by another thread/process, and is not available." .format(key))
def __init__(self, ns: str, name: str, expiration: Union[int, float], redis_backend: RedisBackend) -> None: super().__init__(ns, name) self.__redis = redis_backend.get_redis_connection(ns) with _map_to_sdl_exception(): redis_lockname = '{' + ns + '},' + self._lock_name self.__redis_lock = Lock(redis=self.__redis, name=redis_lockname, timeout=expiration) self._register_scripts()
def redis_lock_from_event(event): from django_redis import get_redis_connection from redis.lock import Lock if not hasattr(event, '_lock') or not event._lock: rc = get_redis_connection("redis") event._lock = Lock(redis=rc, name='pretix_event_%s' % event.id, timeout=LOCK_TIMEOUT) return event._lock
def set_hash_map_cache(name, mapping, ex=None): """ name : name mapping : mapping dict ex : 超期时间 (单位:秒) """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(name), timeout=20): if DEBUG_CACHE: print "set redis hash map. %s %s" % (name, ex) cli.hmset(name, mapping) _set_expire_time(cli, name, ex) return True
def greenlet_scheduler(self): redis_scheduler_lock_key = "%s:schedulerlock" % get_current_config( )["redis_prefix"] while True: with Lock(connections.redis, redis_scheduler_lock_key, timeout=self.config["scheduler_interval"] + 10, blocking=False, thread_local=False): self.scheduler.check() time.sleep(self.config["scheduler_interval"])
def wrapper(*args, **kwargs): init_transaction() with Lock(redis_lock, get_trans_lock_name(), timeout=5) as look: try: begin_transaction() result = func(*args, **kwargs) end_transaction() close_transaction(get_transaction_id()) return result except Exception as e: rollback(get_transaction_id()) raise e finally: release_transaction()
def __enter__(self): self.lock = Lock( redis_client, self.lock_name, blocking_timeout=1, timeout=self.lock_timeout, ) try: self.is_lock_free = self.lock.acquire(blocking=False) except LockError: log.error( 'lock acquire error', extra={'data': {'lock_name': self.lock_name}}, exc_info=True, ) return self
def set_cache(key, value, ex=None): """ key : redis 键名 value : redis 键值 ex : 超期时间 (单位:秒) """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(key), timeout=20): if DEBUG_CACHE: print "set redis cache. %s %s" % (key, ex) try: v = JSONDataEncoder.encode(value) except: v = value cli.set(key, v, ex) return True
def end(self): if self.p is not None: self.p.execute() lockname = self.rollout.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'got lock {lockname}') if not self.redis.exists(self.rollout.key('finalized')): self.redis.lpush(self.rollout.key('episodes'), self.id) self.redis.incrby(self.rollout.key('steps'), len(self)) if lk.owned(): lk.release()
def raise_or_lock(self, key, timeout): """ Checks if the task is locked and raises an exception, else locks the task. By default, the tasks and the key expire after 60 minutes. (meaning it will not be executed and the lock will clear). """ acquired = Lock(self.redis, key, timeout=timeout, blocking=self.blocking, blocking_timeout=self.blocking_timeout).acquire() if not acquired: # Time remaining in milliseconds # https://redis.io/commands/pttl ttl = self.redis.pttl(key) raise AlreadyQueued(ttl / 1000.)
def redis_lock(name: str, **kwargs) -> Generator: """ Acquire a Redis lock. This is a wrapper around redis.lock.Lock(), that also works in tests (there, the lock is always granted without any checks). Relevant kwargs are: - blocking_timeout: how many seconds to try to acquire the lock. Use 0 for a non-blocking lock. The default is None, which means we wait forever. - timeout: how many seconds to keep the lock for. The default is None, which means it remains locked forever. Raises redis.exceptions.LockError if the lock couldn't be acquired or released. """ if settings.DJTRIGGERS_REDIS_URL.startswith('redis'): # pragma: no cover with Lock(redis=Redis.from_url(settings.DJTRIGGERS_REDIS_URL), name=name, **kwargs): yield else: yield
def set_hash_cache(name, key, value, ex=None): """ name : redis hash name key : redis 键名 value : redis 键值 ex : 超期时间 (单位:秒) """ cli = RedisPool().connection() with Lock(cli, _dist_lock_key(name), timeout=20): if DEBUG_CACHE: print "set redis hash cache. %s %s" % (key, ex) try: v = JSONDataEncoder.encode(value) except: v = value cli.hset(name, key, v) _set_expire_time(cli, name, ex) return True
def __init__(self, persist_mode=False, key_prefix='', min_cache_time=5, force_cache_time=False, base='store', path='cache', redis_host='localhost', redis_port=6379, redis_db=1, redis_file=None): self.__key_prefix = key_prefix self.__cache_key = '{}:cache'.format(key_prefix) self.__persist_mode = persist_mode self.__min_cache_time = min_cache_time self.__force_cache_time = force_cache_time self.__base_path = base self.__resource_cache = get_triple_store() self._r = get_kv(persist_mode, redis_host, redis_port, redis_db, redis_file, base=base, path=path) self.__lock = Lock(self._r, key_prefix) self.__mlock = TLock() self.__memory_graphs = {} self.__memory_order = [] self.__resources_ts = {} # Clean temporal folders under 'base' (others than 'path' subfolder) for sub in filter(lambda x: x != path, get_immediate_subdirectories(base)): shutil.rmtree('{}/{}'.format(self.__base_path, sub)) for lock_key in self._r.keys('{}:l*'.format(self.__key_prefix)): self._r.delete(lock_key) self.__enabled = True self.__purge_th = Thread(target=self.__purge) self.__purge_th.daemon = True self.__purge_th.start()
def __init__(self, persist_mode=None, key_prefix='', min_cache_time=5, force_cache_time=False, base='store', path='cache', redis_host='localhost', redis_port=6379, redis_db=1, redis_file=None, graph_memory_limit=5000): self.__key_prefix = key_prefix self.__cache_key = '{}:cache'.format(key_prefix) self.__persist_mode = persist_mode self.__min_cache_time = min_cache_time self.__force_cache_time = force_cache_time self.__base_path = base self._r = get_kv(persist_mode, redis_host, redis_port, redis_db, redis_file, base=base, path=path) self.__lock = Lock(self._r, key_prefix) self.__mlock = TLock() self.__graph_memory_limit = graph_memory_limit self.__memory_graphs = {} self.__memory_order = [] self.__resources_ts = {} for lock_key in self._r.keys('{}:l*'.format(self.__key_prefix)): self._r.delete(lock_key) self._r.delete(key_prefix) self.__enabled = True self.__purge_th = Thread(target=self.__purge) self.__purge_th.daemon = True self.__purge_th.start()
def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs): """ :param unique_id: An identifier for this auth object. Auth instances which wish to share tokens must use the same ID. :type unique_id: `unicode` :param redis_server: An instance of a Redis server, configured to talk to Redis. :type redis_server: :class:`Redis` """ # pylint:disable=keyword-arg-before-vararg self._unique_id = unique_id self._redis_server = redis_server or StrictRedis() refresh_lock = Lock(redis=self._redis_server, name='{0}_lock'.format(self._unique_id)) super(RedisManagedOAuth2Mixin, self).__init__(*args, refresh_lock=refresh_lock, **kwargs) if self._access_token is None: self._get_and_update_current_tokens()
def _send_smses(send_deferred=False, backend=None, limit=None): # Get lock so there is only one sms sender at the same time. if send_deferred: send_lock_name = 'smsgateway_send_sms_deferred' else: send_lock_name = 'smsgateway_send_sms' with Lock(redis=Redis.from_url(settings.SMSGATEWAY_REDIS_URL), name='smsgateway-' + send_lock_name, blocking_timeout=0): successes, failures = 0, 0 try: # Get SMSes that need to be sent (deferred or non-deferred) if send_deferred: to_send = QueuedSMS.objects.filter(priority=PRIORITY_DEFERRED) else: to_send = QueuedSMS.objects.exclude(priority=PRIORITY_DEFERRED) if isinstance(limit, int): to_send = to_send[:limit] # Send each SMS for sms in to_send: if backend: sms_using = backend else: sms_using = None if sms.using == '__none__' else sms.using if send(sms.to, sms.content, sms.signature, sms_using, sms.reliable): # Successfully sent, remove from queue sms.delete() successes += 1 else: # Failed to send, defer SMS sms.defer() failures += 1 finally: if successes and failures: statsd.gauge('smsgateway.success_rate', successes / failures) else: statsd.gauge('smsgateway.success_rate', 1)
def greenlet_queuestats(self): interval = min(self.config["orchestrate_interval"], 1 * 60) lock_timeout = 5 * 60 + (interval * 2) while True: lock = Lock(connections.redis, self.redis_queuestats_lock_key, timeout=lock_timeout, thread_local=False, blocking=False) with lock: lock_expires = time.time() + lock_timeout self.queue_etas = defaultdict(lambda: MovingETA(5)) while True: self.queuestats() # Because queue stats can be expensive, we try to keep the lock on the same agent lock_extend = (time.time() + lock_timeout) - lock_expires lock_expires += lock_extend lock.extend(lock_extend) time.sleep(interval) time.sleep(interval)
def request(redis, method, url, json_): """Request wrapper to handle errors""" logging.info("[UTILS REQUEST] method: %s url: %s json: %s", method, url, json_) response = requests.request( method, urljoin(API_CALLBACK, url), json=json_, timeout=30, headers={"Authorization": f"processing-token {PROCESSING_TOKEN}"}, ) if 400 <= response.status_code < 500: # client error, log and fix if necessary logging.error(response.text) capture_message(response.text) elif response.status_code >= 500: # server error, store for retry redis.lpush( redis_fields.error_retry_queue(), json.dumps({"method": method, "url": url, "json": json_}), ) logging.info( "[UTILS REQUEST] queueing for retry: method: %s url: %s json: %s", method, url, json_, ) logging.error(response.text) capture_message(response.text) elif ( 200 <= response.status_code < 300 and not Lock(redis, redis_fields.error_retry_lock()).locked() and redis.llen(redis_fields.error_retry_queue()) > 0 ): # success, retry error queue if populated publisher.publish(RETRY_ERROR_TOPIC, encode_pubsub_data({})) return response
def testRedisNativeLock(db): l = Lock(db.redis, 'zelock', timeout=3.0) l.acquire() l.release()
from flask_sqlalchemy import SQLAlchemy from .alchemy_encoder import AlchemyEncoder from werkzeug.contrib.cache import RedisCache from redis.lock import Lock from flask_bcrypt import Bcrypt from flask_assets import Environment, Bundle # set up application application = Flask(__name__) application.config.from_envvar('APP_CONFIG') application.json_encoder = AlchemyEncoder application.cache = RedisCache(application.config['REDIS_HOST'], application.config['REDIS_PORT'], \ default_timeout=application.config['CACHE_TIMEOUT']) application.cache_lock = Lock(application.cache._client, 'mgtrk_worker_lock', timeout=0.2, blocking_timeout=0.2) # set up authentication bcrypt = Bcrypt(application) # bundle js and css code assets = Environment(application) js = Bundle('js/core/*', filters='rjsmin', output='js/core/mgtrk-core.min.js') assets.register('core-js', js) css = Bundle('css/core/*', filters='cssmin', output='css/core/mgtrk-core.min.css') assets.register('core-css', css)
def uri_lock(self, uri): with self.__lock: key = '{}:l:'.format(self.__key_prefix) + uri key = (key[:250]) if len(key) > 250 else key return Lock(self._r, key)
def _redis_lock(self) -> Lock: if self._memoised_lock is None: self._memoised_lock = Lock(self.redis, self._lock_name) return self._memoised_lock
from headphones2.exceptions import HeadphonesException from headphones2.external.musicbrainz import get_release_groups_for_artist, get_releases_for_release_group, \ musicbrainz_lock from headphones2.orm import Artist, Status, Album, Release, Track, MediaFile from headphones2.taggers.pipeline import match_album_from_list_of_paths from headphones2.utils.general import ensure_unicode from headphones2.utils.structs import FolderResult from redis.lock import Lock from .engine import huey from headphones2 import local_redis from ..orm import connect logger = logbook.Logger(__name__) write_lock = Lock(local_redis, name='sqlite_write') @huey.task() def add_artist_task(artist_id): with closing(connect()) as session: already_working = local_redis.setnx(artist_id, 'in_progress') if already_working == 1: add_artist_to_db(artist_id, session) local_redis.delete(artist_id) return True else: logger.debug('Task for adding artist {} already in progress'.format(artist_id)) return False