def add_root(self, pipe: Redis, root: RedisKey) -> None:
"""Add the content to be cleaned up.
Args:
pipe: Piped redis commands.
root: The redis key prefix.
"""
cleanup_val = root.wildcard
pipe.rpush(self.keys.roots, cleanup_val)
def add_namespace(self, pipe: Redis, keys: NamespaceKeys) -> None:
"""Add the namespace to be cleaned up.
Args:
pipe: Piped redis commands.
keys: The namespace key group.
"""
cleanup_val = keys.root.named['namespace']
pipe.rpush(self.keys.namespaces, cleanup_val)
def add_content(self, pipe: Redis, keys: ContentKeys) -> None:
"""Add the content to be cleaned up.
Args:
pipe: Piped redis commands.
keys: The content key group.
"""
namespace = keys.root.named['namespace']
email_id = keys.root.named['email_id']
cleanup_val = b'%b\x00%b' % (namespace, email_id)
pipe.rpush(self.keys.contents, cleanup_val)
def add_mailbox(self, pipe: Redis, keys: MailboxKeys) -> None:
"""Add the mailbox to be cleaned up.
Args:
pipe: Piped redis commands.
keys: The mailbox key group.
"""
namespace = keys.root.named['namespace']
mailbox_id = keys.root.named['mailbox_id']
cleanup_val = b'%b\x00%b' % (namespace, mailbox_id)
pipe.rpush(self.keys.mailboxes, cleanup_val)
def add_message(self, pipe: Redis, keys: MessageKeys) -> None:
"""Add the message to be cleaned up.
Args:
pipe: Piped redis commands.
keys: The message key group.
"""
namespace = keys.root.named['namespace']
mailbox_id = keys.root.named['mailbox_id']
msg_uid = keys.root.named['uid']
cleanup_val = b'%b\x00%b\x00%b' \
% (namespace, mailbox_id, msg_uid)
pipe.rpush(self.keys.messages, cleanup_val)
async def reset_redis(redis: Redis,
loop: AbstractEventLoop,
hard: bool = False) -> None:
if hard:
await redis.flushall()
else:
await redis.delete(q_key, info_key, seen_key, scope_key, done_key),
await redis.hset(info_key, "crawl_depth", 2),
for url in default_seed_list:
await asyncio.gather(
redis.rpush(q_key, ujson.dumps({
"url": url,
"depth": 0
})),
redis.sadd(seen_key, url),
loop=loop,
)