def push_to_queue(self, priority, filter_task):
task = PriorityTask(priority, filter_task)
while True:
try:
self.filter_task_queue.put_nowait(task)
break
except queue.Full:
logger.warning("FilterTask队列已满，1秒后重试.")
self.ev.wait(1)
continue
def _worker(self):
logger.info("RefreshEngine start!")
refresh_task_queue = self.app_ctx.MessageQueues.SEARCH_TASK_QUEUE
while self.status == self.EngineStatus.RUNNING:
logger.debug("start build search task.")
rows = GeyeSearchRuleModel.objects.filter(is_deleted=0,
status=1).all()
current_time = datetime.datetime.now()
for row in rows:
delay = int(row.delay)
if row.last_refresh_time + datetime.timedelta(
minutes=delay) < current_time:
# 该刷新了，添加到任务队列中去
# 添加一个字典，如果后续改成分布式，需要改成JSON字符串
# Task格式：
# tuple(priority, _task)
# build task
_data = {
"search_rule_id": row.id,
"search_rule_name": row.name,
"search_rule_content": row.rule,
}
# task = (row.priority, _data)
task = PriorityTask(row.priority, _data)
logger.debug("task: {}".format(task))
while True:
try:
refresh_task_queue.put_nowait(task)
break
except queue.Full:
logger.warning("SearchTask队列已满，等待3秒后重试")
self.ev.wait(3)
continue
# 更新任务的最后刷新时间
row.last_refresh_time = current_time
row.save()
self.ev.wait(settings.REFRESH_INTERVAL)
logger.info("RefreshEngine end!")
def make_request(self, header, data) -> Optional[requests.Response]:
"""
发出搜索请求
:param header: 请求的header，包括token等信息
:param data: 搜索的内容
"""
# 获取代理设置信息
proxies = random.choice(self.all_proxies) if self.use_proxies else None
# 请求计数
# todo：先写死到代码里，计划移植到配置中
request_cnt = 0
while self.is_running():
try:
request_cnt += 1
if request_cnt == 5:
logger.warning("请求超出最大次数!")
break
logger.debug("before requests.get()")
response = requests.get(self.search_api_url,
params=data,
headers=header,
timeout=12,
proxies=proxies)
logger.debug("after requests.get()")
return response
except requests.RequestException as e:
logger.error(
"Error while make request. requests.RequestException: {}".
format(e))
logger.error("Try re-request after 5s.")
self.ev.wait(5)
continue
return None