def do_task(self, task): """Return the result of the task or raise a TaskFailed exception on failure.""" def _on_fail(): raise self.TaskFailed("Task failed") task.on_fail.append(_on_fail) taskset = Taskset([task]) if not self.do_taskset(taskset, timeout=task.timeout): raise self.TaskFailed("Task timeout") return task.result
def dispatch_background_task(self, func, arg, uniq=None, high_priority=False): """Submit a background task and return its handle.""" task = Task(func, arg, uniq, background=True, high_priority=high_priority) taskset = Taskset([task]) self.do_taskset(taskset) return task.handle
def do_task(self, task): """Return the result of the task or raise a TaskFailed exception on failure.""" rtn = defer.Deferred() def _on_fail(): rtn.errback(self.TaskFailed("Task failed")) task.on_fail.append(_on_fail) task.on_complete.append(rtn.callback) taskset = Taskset([task]) taskset.handles = {} c = self._submit_task(task) taskset.connections = set([c]) def _do_cmd(*args): return self._command_handler(taskset, c, *args) with _ConnectionWrapper(c, _do_cmd): rtn = yield rtn defer.returnValue(rtn)
def run_worker(worker, arg, **kwargs): discover_workers() task = Task(worker, arg, **kwargs) if client: if task.background: taskset = Taskset([task]) try: client.do_taskset(taskset) except GearmanConnection.ConnectionError: # try again, perhaps server connection was reset client.do_taskset(taskset) return task.handle else: return client.do_task(task) else: if workers.has_key(worker): return workers[worker](task) else: raise NotImplementedError()
urls = [] con = sqlite3.connect(db) for tb in CATES: sql = 'select * from %s limit %d' % (tb, TRAIN_URLS) rows = con.execute(sql) urls.extend(['%s\t%s' % (row[0].encode('utf-8'), tb) for row in rows]) pass return urls if __name__ == '__main__': urls = get_urls(URLS_DB) client = GearmanClient(['10.61.0.145']) tasks = Taskset() TASK_URLS_NUM = 100 # disptribute task i = 0 while i < len(urls): sub_urls = urls[i:i + TASK_URLS_NUM] workload = '\n'.join(sub_urls) t = Task1('crawl', workload, str(i), timeout=TASK1_TIMEOUT, retry_count=1) tasks.add(t) print "add task:%s" % t.uniq i += TASK_URLS_NUM # test
urls = [] con = sqlite3.connect(db) for tb in CATES: sql = "select * from %s limit %d" % (tb, TRAIN_URLS) rows = con.execute(sql) urls.extend(["%s\t%s" % (row[0].encode("utf-8"), tb) for row in rows]) pass return urls if __name__ == "__main__": urls = get_urls(URLS_DB) client = GearmanClient(["10.61.0.145"]) tasks = Taskset() TASK_URLS_NUM = 100 # disptribute task i = 0 while i < len(urls): sub_urls = urls[i : i + TASK_URLS_NUM] workload = "\n".join(sub_urls) t = Task1("crawl", workload, str(i), timeout=TASK1_TIMEOUT, retry_count=1) tasks.add(t) print "add task:%s" % t.uniq i += TASK_URLS_NUM # test pass # 0.init database for return result from worker print "0.initialize database for results."