def push_to_queue(self, priority, filter_task):
task = PriorityTask(priority, filter_task)
while True:
try:
self.filter_task_queue.put_nowait(task)
break
except queue.Full:
logger.warning("FilterTask队列已满，1秒后重试.")
self.ev.wait(1)
continue
def __put_task(self, priority, task):
"""
放置任务到队列中
:param priority:
:param task:
:return:
"""
wrap_task = PriorityTask(priority, task)
while self.__is_running():
try:
self._monitor_result_queue.put_nowait(wrap_task)
return
except queue.Full:
self.ev.wait(1)
continue
def _worker(self):
logger.info("RefreshEngine start!")
refresh_task_queue = self.app_ctx.MessageQueues.SEARCH_TASK_QUEUE
while self.status == self.EngineStatus.RUNNING:
logger.debug("start build search task.")
rows = GeyeSearchRuleModel.objects.filter(is_deleted=0,
status=1).all()
current_time = datetime.datetime.now()
for row in rows:
delay = int(row.delay)
if row.last_refresh_time + datetime.timedelta(
minutes=delay) < current_time:
# 该刷新了，添加到任务队列中去
# 添加一个字典，如果后续改成分布式，需要改成JSON字符串
# Task格式：
# tuple(priority, _task)
# build task
_data = {
"search_rule_id": row.id,
"search_rule_name": row.name,
"search_rule_content": row.rule,
}
# task = (row.priority, _data)
task = PriorityTask(row.priority, _data)
logger.debug("task: {}".format(task))
while True:
try:
refresh_task_queue.put_nowait(task)
break
except queue.Full:
logger.warning("SearchTask队列已满，等待3秒后重试")
self.ev.wait(3)
continue
# 更新任务的最后刷新时间
row.last_refresh_time = current_time
row.save()
self.ev.wait(settings.REFRESH_INTERVAL)
logger.info("RefreshEngine end!")
def _real_worker(self):
while self.is_running():
priority, task = self._get_task()
if not priority or not task:
continue
# 解析任务内容
# "rule_id": row.id,
# "rule_content": row.rule,
rule_content = task.get("rule_content")
rule_id = task.get("rule_id")
if not rule_content:
continue
# 构建请求的参数
# 有几个参数是必须的
# p 是页码，q是搜索词
params = {
"tab": "public",
"scope": "/",
"type": "content",
"q": self.__encode_keyword(rule_content)
}
# 默认搜索 3 页
results = []
for p in range(1, 4):
params["p"] = p
# 从 db 里取出 yuque.com 域名的 cookie 信息
# 先写死 "yuque.com"
header = {
"Cookie": self._get_cookie_header()
}
logger.debug("yuque header: {}".format(header))
response = self.make_request(self.SEARCH_API, header, params)
parsed_result = self.parse_response(response)
results.extend(parsed_result)
# 放进队列中
self._put_task(PriorityTask(priority, {"rule_id": rule_id, "results": results}))
def _worker(self):
self.logger.info(f"{self.name} start!")
while self.is_running():
self.logger.debug("start build yuque search task.")
rows: Iterable[
GeyeYuqueSearchRuleModel] = GeyeYuqueSearchRuleModel.instance.all_enable_rules(
)
for row in rows:
interval = row.interval
last_refresh_time = row.last_refresh_time
# 判断是否存在需要生成任务的关键字
if last_refresh_time + datetime.timedelta(
minutes=interval) > datetime.datetime.now():
continue
# 生成搜索任务
task = PriorityTask(row.priority, {
"rule_id": row.id,
"rule_content": row.rule,
})
self.logger.debug(f"Create yuque search task: {task}")
# 添加到任务队列
while self.is_running():
try:
self._task_queue.put_nowait(task)
break
except queue.Full:
self.ev.wait(3)
continue
self.ev.wait(30)
self.logger.info(f"{self.name} stop!")
def _worker(self):
logger.info("{name} start!".format(name=self.name))
while self.__running():
logger.debug("start build monitor task.")
rows: List[GeyeMonitorRules] = GeyeMonitorRules.instance.get_all()
current_time = datetime.datetime.now()
for _row in rows:
interval = _row.interval
if _row.last_fetch_time + datetime.timedelta(
minutes=interval) < current_time:
task = PriorityTask(
_row.priority, {
"task_type": _row.task_type,
"event_type": _row.event_type,
"rule_content": _row.rule_content,
"rule_id": _row.id,
})
logger.debug(
"Create monitor task: {task}".format(task=task))
while self.__running():
try:
self._monitor_task_queue.put_nowait(task)
break
except queue.Full:
self.ev.wait(3)
continue
# 更新rule的最后刷新时间
_row.last_fetch_time = current_time
_row.save()
self.ev.wait(30)
logger.info("{name} stop!".format(name=self.name))
def put_task_to_queue(self,
task,
target_queue: queue.PriorityQueue = None):
"""
把任务放到队列中去
:param task: 待处理的任务
:param target_queue: 待放入的队列
:return:
"""
if not target_queue:
target_queue = self.filter_task_queue
while self.status == self.EngineStatus.RUNNING:
try:
target_queue.put_nowait(PriorityTask(task[0], task[1]))
break
except queue.Full:
# get queue name
q_name = "unknown"
for k, v in self.__dict__.items():
if v is target_queue:
q_name = k
logger.debug("{q_name}已满，1秒后重试.".format(q_name=q_name))
self.ev.wait(1)
continue
:homepage: None
:license: GPL-3.0, see LICENSE for more details.
:copyright: Copyright (c) 2017-2018 lightless. All rights reserved
"""
import sys
import os
sys.path.append("..")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geye.settings")
import django
django.setup()
from geye.application import GeyeApplication
from geye.database.models.monitorRules import MonitorTaskTypeConstant, MonitorEventTypeConstant
from geye.utils.datatype import PriorityTask
if __name__ == '__main__':
app = GeyeApplication("test")
app.start(queues=["monitor_task_queue"],
engines=["monitor_refresh_engine", "monitor_engine"])
app.MessageQueues.MONITOR_TASK_QUEUE.put(
PriorityTask(
5, {
"task_type": MonitorTaskTypeConstant.USER,
"event_type": MonitorEventTypeConstant.PUSH_EVENT,
"rule_content": '{"username": "******"}',
}))