def queue_post_watch_cleanup(uowm: UnitOfWorkManager, config: Config) -> NoReturn:
"""
Send all watches to celery to check if the post has been deleted
:param uowm: Unit of work manager
"""
print('[Scheduled Job] Queue Deleted Watch Check')
redis = get_redis_client(config)
if len(redis.lrange('watch_remove_deleted', 0, 20000)) > 0:
log.info('Deleted watchqueue still has pending jobs. Skipping update queueing ')
return
with uowm.start() as uow:
watches = uow.repostwatch.get_all()
for chunk in chunk_list(watches, 30):
check_if_watched_post_is_active.apply_async((chunk,))
def queue_config_updates(uowm: UnitOfWorkManager, config: Config) -> NoReturn:
print('[Scheduled Job] Queue config update check')
redis = get_redis_client(config)
if len(redis.lrange('config_update_check', 0, 20000)) > 0:
log.info(
'Config update queue still has pending jobs. Skipping update queueing '
)
return
with uowm.start() as uow:
monitored_subs = uow.monitored_sub.get_all()
for monitored_sub in monitored_subs:
check_for_subreddit_config_update_task.apply_async(
(monitored_sub, ))
print('[Scheduled Job Complete] Queue config update check')
def get_all_links():
print('[Scheduled Job] Queue deleted posts')
redis = get_redis_client(config)
if len(redis.lrange('deleted_post_cleanup', 0, 20000)) > 0:
log.info(
'Deleted post cleanup queue still has pending jobs. Skipping queueing '
)
return
conn = get_db_conn()
batch = []
with conn.cursor() as cur:
query = f"SELECT post_id, url, post_type FROM reddit_post WHERE post_type='image' AND last_deleted_check <= NOW() - INTERVAL 90 DAY LIMIT 1000000"
cur.execute(query)
log.info('Adding items to index')
for row in cur:
batch.append({'id': row['post_id'], 'url': row['url']})
if len(batch) >= 30:
try:
deleted_post_cleanup.apply_async(
(batch, ), queue='deleted_post_cleanup')
batch = []
except Exception as e:
continue
response_builder = ResponseBuilder(uowm)
reddit = get_reddit_instance(config)
reddit_manager = RedditManager(reddit)
dup = DuplicateImageService(uowm, event_logger, reddit, config=config)
monitor = SubMonitor(dup,
uowm,
reddit_manager,
response_builder,
ResponseHandler(reddit_manager,
uowm,
event_logger,
source='submonitor',
live_response=config.live_responses),
event_logger=event_logger,
config=config)
redis = get_redis_client(config)
while True:
while True:
queued_items = redis.lrange('submonitor', 0, 20000)
if len(queued_items) == 0:
log.info('Sub monitor queue empty. Starting over')
break
log.info('Sub monitor queue still has %s tasks', len(queued_items))
time.sleep(60)
with uowm.start() as uow:
monitored_subs = uow.monitored_sub.get_all()
for monitored_sub in monitored_subs:
if not monitored_sub.active:
continue
log.info('Checking sub %s', monitored_sub.name)
if not monitored_sub.active: