def do_task(self, task):
"""Return the result of the task or raise a TaskFailed exception on failure."""
rtn = defer.Deferred()
def _on_fail():
rtn.errback(self.TaskFailed("Task failed"))
task.on_fail.append(_on_fail)
task.on_complete.append(rtn.callback)
taskset = Taskset([task])
taskset.handles = {}
c = self._submit_task(task)
taskset.connections = set([c])
def _do_cmd(*args):
return self._command_handler(taskset, c, *args)
with _ConnectionWrapper(c, _do_cmd):
rtn = yield rtn
defer.returnValue(rtn)
urls = []
con = sqlite3.connect(db)
for tb in CATES:
sql = "select * from %s limit %d" % (tb, TRAIN_URLS)
rows = con.execute(sql)
urls.extend(["%s\t%s" % (row[0].encode("utf-8"), tb) for row in rows])
pass
return urls
if __name__ == "__main__":
urls = get_urls(URLS_DB)
client = GearmanClient(["10.61.0.145"])
tasks = Taskset()
TASK_URLS_NUM = 100
# disptribute task
i = 0
while i < len(urls):
sub_urls = urls[i : i + TASK_URLS_NUM]
workload = "\n".join(sub_urls)
t = Task1("crawl", workload, str(i), timeout=TASK1_TIMEOUT, retry_count=1)
tasks.add(t)
print "add task:%s" % t.uniq
i += TASK_URLS_NUM
# test
pass
# 0.init database for return result from worker
print "0.initialize database for results."
