Ejemplo n.º 1
0
class TaskUtil:
    '''
    Task操作工具
    tasks
    {
        "parser": "phase"
        "request": "http://so.eduyun.cn/synResource",
        "response": "<html>...</html>"
        "parent": {},
        "state": "done",
        "uptime":
    }
    '''
    def __init__(self, request):
        self.mongoUtil = MongoUtil()
        count = self.mongoUtil.count(collection_name='tasks')
        if count == 0:
            task = None
            task_file = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), '..', 'conf',
                'task.json')
            f = open(task_file, encoding='utf-8')
            task = f.read()
            f.close()
            insert_data = json.loads(task)
            self.mongoUtil.insert(collection_name='tasks',
                                  insert_data=insert_data)

    def get_ready(self):
        '''
        获取一条待执行的任务(准备状态),并置为doing状态
        :return: dict 单条任务
        '''
        # 过滤条件,不存在state字段或state=ready
        filter_dict = {
            '$or': [{
                'state': {
                    '$exists': False
                }
            }, {
                'state': 'ready'
            }]
        }
        # 更新条件,将state=doing
        update_dict = {'$set': {'state': 'doing'}}
        # 执行mongo操作
        task = self.mongoUtil.find_one_and_update(collection_name='tasks',
                                                  filter_dict=filter_dict,
                                                  update_dict=update_dict)
        return task

    def set_state(self, id, state):
        '''
        设置任务状态(ready,doing,done)
        :param id: str 主键id
        :param state: str 更新状态值
        :return: 无
        '''
        filter_dict = {'_id': id}
        update_dict = {'$set': {'state': state}}
        self.mongoUtil.update(collection_name='tasks',
                              filter_dict=filter_dict,
                              update_dict=update_dict)

    def replace_one(self, id, task):
        '''
        更新整个任务
        :param id: str
        :param task: dict
        :return: 无
        '''
        filter_dict = {'_id': id}
        r = self.mongoUtil.find_one_and_replace(collection_name='tasks',
                                                filter_dict=filter_dict,
                                                replace_dict=task)
        return r

    def insert_one(self, task):
        '''
        插入一条task
        :param parser: str 解析器
        :param request: str 请求的url
        :return: 无
        '''
        r = self.mongoUtil.find_one(collection_name='tasks', filter_dict=task)
        if r is None:
            task['state'] = 'ready'
            r = self.mongoUtil.insert(collection_name='tasks',
                                      insert_data=task)

    def __del__(self):
        self.mongoUtil.close_conn()
Ejemplo n.º 2
0
class TaskUtil:
    '''
    Task操作工具
    tasks
    {
        "parser": "phase"
        "request": "http://so.eduyun.cn/synResource",
        "response": "<html>...</html>"
        "parent": {},
        "state": "done",
        "uptime":
    }
    '''
    def __init__(self):
        self.mongoUtil = MongoUtil()
        count = self.mongoUtil.count(collection_name='tasks')
        if count == 0:
            first_task_parser = Setting.FIRST_TASK_PARSER
            first_task_url = Setting.FIRST_TASK_URL
            first_task_table = Setting.FIRST_TASK_TABLE
            insert_data = {
                "parser": first_task_parser,
                "request": first_task_url,
                "table": first_task_table,
                "parent": {},
                "state": "ready"
            }
            self.mongoUtil.insert(collection_name='tasks',
                                  insert_data=insert_data)

    def get_ready(self):
        '''
        获取一条待执行的任务(准备状态),并置为doing状态
        :return: dict 单条任务
        '''
        # 过滤条件,不存在state字段或state=ready
        filter_dict = {
            '$or': [{
                'state': {
                    '$exists': False
                }
            }, {
                'state': 'ready'
            }]
        }
        # 更新条件,将state=doing
        update_dict = {'$set': {'state': 'doing'}}
        # 执行mongo操作
        task = self.mongoUtil.find_one_and_update(collection_name='tasks',
                                                  filter_dict=filter_dict,
                                                  update_dict=update_dict)
        return task

    def set_state(self, id, state):
        '''
        设置任务状态(ready,doing,done)
        :param id: str 主键id
        :param state: str 更新状态值
        :return: 无
        '''
        filter_dict = {'_id': id}
        update_dict = {'$set': {'state': state}}
        self.mongoUtil.update(collection_name='tasks',
                              filter_dict=filter_dict,
                              update_dict=update_dict)

    def replace_one(self, id, task):
        '''
        更新整个任务
        :param id: str
        :param task: dict
        :return: 无
        '''
        filter_dict = {'_id': id}
        r = self.mongoUtil.find_one_and_replace(collection_name='tasks',
                                                filter_dict=filter_dict,
                                                replace_dict=task)
        return r

    def insert_one(self, task):
        '''
        插入一条task
        :param parser: str 解析器
        :param request: str 请求的url
        :return: 无
        '''
        r = self.mongoUtil.find_one(collection_name='tasks', filter_dict=task)
        if r is None:
            task['state'] = 'ready'
            r = self.mongoUtil.insert(collection_name='tasks',
                                      insert_data=task)

    def __del__(self):
        self.mongoUtil.close_conn()