def iter_expectations_ad_accounts(sweep_id: str) -> Generator[str, None, None]:
"""
Yields ad account IDs for which we have expectations
"""
redis = get_redis()
sweep_level_key = _sweep_aa_index_key_template(sweep_id=sweep_id)
for ad_account_id in redis.sscan_iter(sweep_level_key):
yield ad_account_id.decode('utf8')
def iter_expectations_per_ad_account(
ad_account_id: str, sweep_id: str) -> Generator[str, None, None]:
"""
Yield JobIDs corresponding to expecations set per this AA ID, SweepID
"""
redis = get_redis()
aa_level_key = _aa_job_index_key_template(sweep_id=sweep_id,
ad_account_id=ad_account_id)
for job_id in redis.sscan_iter(aa_level_key):
yield job_id.decode('utf8')
def __init__(self, sweep_id: str, cache_max_size: Optional[int] = 2000):
self.sweep_id = sweep_id
self._aa_cache = set()
self._aa_cache_order = deque()
self._aa_cache_remaining = cache_max_size
self._job_id_cache = set()
self._job_id_cache_order = deque()
self._job_id_cache_remaining = cache_max_size
self._redis_client = get_redis()
def __init__(self, asset_scope: str, sweep_id: str):
self.queue_key = f'{asset_scope}-{sweep_id}-sorted-token-queue'
self._redis = get_redis()
def report_task_done(task_id: TaskID):
shard_key, task_key = task_id
get_redis().hdel(shard_key, task_key)
def report_task_active(task_id: TaskID):
shard_key, task_key = task_id
get_redis().hset(shard_key, task_key, time.time())
def get_task_data(task_id: TaskID):
return get_redis().hget(*task_id)
def get_remaining_tasks_count(self):
redis = get_redis()
counts = gevent.pool.Pool(size=self.number_of_shards).imap_unordered(
lambda shard_id: redis.hlen(self._generate_shard_key(shard_id)) or
0, range(self.number_of_shards))
return sum(counts)
def is_set(cls, sweep_id: str) -> bool:
return bool(get_redis().get(cls._generate_key(sweep_id)))
def __exit__(self, exc_type, exc_val, exc_tb):
get_redis().delete(self._generate_key(self.sweep_id))
def __enter__(self) -> 'SweepRunningFlag':
get_redis().set(self._generate_key(self.sweep_id), 'true')
return self
def __init__(self, sweep_id: str):
self.sweep_id = sweep_id
self.redis = get_redis()
def __init__(self, asset_scope: str, sweep_id: str):
self._redis = get_redis()
self._sweep_id = sweep_id
self._asset_scope = asset_scope
