def iter_expectations_ad_accounts(sweep_id: str) -> Generator[str, None, None]: """ Yields ad account IDs for which we have expectations """ redis = get_redis() sweep_level_key = _sweep_aa_index_key_template(sweep_id=sweep_id) for ad_account_id in redis.sscan_iter(sweep_level_key): yield ad_account_id.decode('utf8')
def iter_expectations_per_ad_account( ad_account_id: str, sweep_id: str) -> Generator[str, None, None]: """ Yield JobIDs corresponding to expecations set per this AA ID, SweepID """ redis = get_redis() aa_level_key = _aa_job_index_key_template(sweep_id=sweep_id, ad_account_id=ad_account_id) for job_id in redis.sscan_iter(aa_level_key): yield job_id.decode('utf8')
def __init__(self, sweep_id: str, cache_max_size: Optional[int] = 2000): self.sweep_id = sweep_id self._aa_cache = set() self._aa_cache_order = deque() self._aa_cache_remaining = cache_max_size self._job_id_cache = set() self._job_id_cache_order = deque() self._job_id_cache_remaining = cache_max_size self._redis_client = get_redis()
def __init__(self, asset_scope: str, sweep_id: str): self.queue_key = f'{asset_scope}-{sweep_id}-sorted-token-queue' self._redis = get_redis()
def report_task_done(task_id: TaskID): shard_key, task_key = task_id get_redis().hdel(shard_key, task_key)
def report_task_active(task_id: TaskID): shard_key, task_key = task_id get_redis().hset(shard_key, task_key, time.time())
def get_task_data(task_id: TaskID): return get_redis().hget(*task_id)
def get_remaining_tasks_count(self): redis = get_redis() counts = gevent.pool.Pool(size=self.number_of_shards).imap_unordered( lambda shard_id: redis.hlen(self._generate_shard_key(shard_id)) or 0, range(self.number_of_shards)) return sum(counts)
def is_set(cls, sweep_id: str) -> bool: return bool(get_redis().get(cls._generate_key(sweep_id)))
def __exit__(self, exc_type, exc_val, exc_tb): get_redis().delete(self._generate_key(self.sweep_id))
def __enter__(self) -> 'SweepRunningFlag': get_redis().set(self._generate_key(self.sweep_id), 'true') return self
def __init__(self, sweep_id: str): self.sweep_id = sweep_id self.redis = get_redis()
def __init__(self, asset_scope: str, sweep_id: str): self._redis = get_redis() self._sweep_id = sweep_id self._asset_scope = asset_scope