def detach(subscribe, beat_sync=False):
"""
@param subscribe:
@param beat_sync: 是否立即删除， True：立即删除，False:节拍同步，随ddt删除
@return:
"""
from faker import Faker
from urllib.parse import urlparse
# 清洗出订阅中的token
token = urlparse(subscribe).path
r = RedisClient().get_driver()
# 遍历所有任务类型
for task in CRAWLER_SEQUENCE:
# 遍历某种类型的链接池
for sub in r.hgetall(REDIS_SECRET_KEY.format(task)).items():
# 匹配用户token
if token == urlparse(sub[0]).path:
# 若节拍同步，立即移除订阅
if beat_sync:
r.hdel(REDIS_SECRET_KEY.format(task), sub[0])
logger.debug(f'>> Detach -> {sub[0]}')
# 否则将订阅过期时间标记为过期，该链接将随下一波任一节点的ddt任务被删除
else:
r.hset(REDIS_SECRET_KEY.format(task), sub[0],
str(Faker().past_datetime()))
break
class SubscribesCleaner(lsu):
"""解耦清洗插件：国内IP调用很可能出现性能滑坡"""
def __init__(self, debug=False):
super(SubscribesCleaner, self).__init__()
self.debug = debug
self.keys = [REDIS_SECRET_KEY.format(s) for s in CRAWLER_SEQUENCE]
self.rc = RedisClient().get_driver()
def offload_task(self):
for key_ in self.keys:
for sub, _ in self.rc.hgetall(key_).items():
self.work_q.put_nowait([sub, key_])
def control_driver(self, sub_info):
try:
node_info: dict = subs2node(sub_info[0], False)
if self.debug:
print(node_info['subs'], node_info['node'].__len__())
if node_info['node'].__len__() <= 3:
self.rc.hdel(sub_info[-1], sub_info[0])
logger.debug(f'>> Detach -> {sub_info[0]}')
except UnicodeDecodeError or TypeError as e:
logger.debug(
f"Retry put the subscribe({sub_info}) to work queue -- {e}")
self.work_q.put_nowait(sub_info)
except Exception as e:
logger.warning(f"{sub_info} -- {e}")
def detach(subscribe, at_once=False):
"""
@param subscribe:
@param at_once: 是否立即删除， True：立即删除，False:节拍同步，随ddt删除
@return:
"""
from faker import Faker
from urllib.parse import urlparse
from config import CRAWLER_SEQUENCE
token = urlparse(subscribe).path
r = RedisClient().get_driver()
for task in CRAWLER_SEQUENCE:
for sub in r.hgetall(REDIS_SECRET_KEY.format(task)).items():
if token == urlparse(sub[0]).path:
if at_once:
r.hdel(REDIS_SECRET_KEY.format(task), sub[0])
else:
r.hset(REDIS_SECRET_KEY.format(task), sub[0],
str(Faker().past_datetime()))
logger.debug(f'>> Detach -> {sub[0]}')
break
def detach(subscribe, at_once=False):
"""
@param subscribe:
@param at_once: 是否立即删除， True：立即删除，False:节拍同步，随ddt删除
@return:
"""
from faker import Faker
from urllib.parse import urlparse
detach_subs = [[REDIS_SECRET_KEY.format(ft[2]), ft[1], str(Faker().past_datetime())] for ft in
FlowTransferStation().fetch_all() if urlparse(subscribe).path in ft[1]]
r = RedisClient().get_driver()
for sub in detach_subs:
if at_once:
r.hdel(sub[0], sub[1])
else:
r.hset(sub[0], sub[1], sub[-1])
logger.debug(f'>> Detach -> {detach_subs}')
class SubscribesCleaner(lsu):
"""解耦清洗插件：国内IP调用很可能出现性能滑坡"""
def __init__(self, debug=False, kill_target: str = None):
super(SubscribesCleaner, self).__init__()
self.debug = debug
self.keys = [REDIS_SECRET_KEY.format(s) for s in CRAWLER_SEQUENCE]
self.rc = RedisClient().get_driver()
self.kill_ = kill_target
def offload_task(self):
for key_ in self.keys:
for sub, _ in self.rc.hgetall(key_).items():
self.work_q.put_nowait([sub, key_])
def killer(self):
"""
@todo redis批量移除或移动hash
@return:
"""
if self.apollo:
for kill_ in self.apollo:
self.rc.hdel(kill_[0], kill_[-1])
logger.debug(f'>> Detach -> {kill_[-1]}')
def control_driver(self, sub_info: List[str]):
"""
@param sub_info: [subs,key_secret_class]
@return:
"""
try:
# 解耦指定簇
if self.kill_ and self.kill_ in sub_info[0]:
self.apollo.append([sub_info[-1], sub_info[0]])
else:
# 解析订阅
node_info: dict = subs2node(sub_info[0], False)
# 打印debug信息
if self.debug:
print(
f"check -- {node_info['subs']} -- {node_info['node'].__len__()}"
)
# 订阅解耦
if node_info['node'].__len__() <= 3:
self.apollo.append([sub_info[-1], sub_info[0]])
except UnicodeDecodeError or TypeError as e:
logger.debug(
f"Retry put the subscribe({sub_info}) to work queue -- {e}")
# 单个链接重试3次,标记超时链接
if self.temp_cache.get(sub_info[0]):
self.temp_cache[sub_info[0]] += 1
else:
self.temp_cache[sub_info[0]] = 1
if self.temp_cache[sub_info[0]] <= 3:
self.work_q.put_nowait(sub_info)
else:
self.apollo.append([sub_info[-1], sub_info[0]])
except Exception as e:
logger.warning(f"{sub_info} -- {e}")