def push_new_task(self, task):
"""插入新的一个task
Args:
task: Task 新的task
Raises:
ScheduleError:当发生错误的时候
"""
if self._is_stopped:
return
try:
if isinstance(task, HttpTask):
if check_http_task_integrity(task):
url = task.request.url if not isinstance(task.request.url, unicode) \
else task.request.url.encode("utf-8")
if not self._processed_url_set.exist(url):
self._prepare_to_process_queue.push(task)
else:
self.logger.debug("request haven been done before.")
else:
self.logger.warn("task is not integrate:%s" % task)
if isinstance(task, FileTask):
self._prepare_to_process_queue.push(task)
except RedisError, e:
raise ScheduleError("redis error in schedule:%s" % e)
def push_new_task(self, task):
"""插入新的一个task
Args:
task: Task 新的task
Raises:
ScheduleError:当发生错误的时候
"""
if self._is_stopped:
return
try:
if isinstance(task, HttpTask):
if check_http_task_integrity(task):
url = task.request.url if not isinstance(task.request.url, unicode) \
else task.request.url.encode("utf-8")
if not self._processed_url_set.exist(url):
self._prepare_to_process_queue.push(task)
else:
self.logger.debug("request haven been done before.")
else:
self.logger.warn("task is not integrate:%s" % task)
if isinstance(task, FileTask):
self._prepare_to_process_queue.push(task)
except RedisError, e:
raise ScheduleError("redis error in schedule:%s" % e)
def push_new_task(self, task):
"""插入新的一个task
Args:
task: Task 新的task
Raises:
ScheduleError:当发生错误的时候
"""
if self._is_stopped:
return
try:
if isinstance(task, HttpTask):
if check_http_task_integrity(task):
self._prepare_to_process_queue.push(task)
else:
self.logger.warn("task is not integrate:%s" % task)
if isinstance(task, FileTask):
self._prepare_to_process_queue.push(task)
except RedisError, e:
raise ScheduleError("redis error in schedule:%s" % e)
