Example #1
0
    def do_task(self, task):
        """Return the result of the task or raise a TaskFailed exception on failure."""

        rtn = defer.Deferred()

        def _on_fail():
            rtn.errback(self.TaskFailed("Task failed"))

        task.on_fail.append(_on_fail)
        task.on_complete.append(rtn.callback)

        taskset = Taskset([task])
        taskset.handles = {}
        
        c = self._submit_task(task)
        taskset.connections = set([c])

        def _do_cmd(*args):
            return self._command_handler(taskset, c, *args)

        with _ConnectionWrapper(c, _do_cmd):
            rtn = yield rtn

        defer.returnValue(rtn)
Example #2
0
    urls = []
    con = sqlite3.connect(db)
    for tb in CATES:
        sql = "select * from %s limit %d" % (tb, TRAIN_URLS)
        rows = con.execute(sql)
        urls.extend(["%s\t%s" % (row[0].encode("utf-8"), tb) for row in rows])
        pass

    return urls


if __name__ == "__main__":
    urls = get_urls(URLS_DB)

    client = GearmanClient(["10.61.0.145"])
    tasks = Taskset()
    TASK_URLS_NUM = 100
    # disptribute task
    i = 0
    while i < len(urls):
        sub_urls = urls[i : i + TASK_URLS_NUM]
        workload = "\n".join(sub_urls)
        t = Task1("crawl", workload, str(i), timeout=TASK1_TIMEOUT, retry_count=1)
        tasks.add(t)
        print "add task:%s" % t.uniq
        i += TASK_URLS_NUM
        # test
        pass

    # 0.init database for return result from worker
    print "0.initialize database for results."