Beispiel #1
0
    def do_task(self, task):
        """Return the result of the task or raise a TaskFailed exception on failure."""
        def _on_fail():
            raise self.TaskFailed("Task failed")

        task.on_fail.append(_on_fail)
        taskset = Taskset([task])
        if not self.do_taskset(taskset, timeout=task.timeout):
            raise self.TaskFailed("Task timeout")
        return task.result
Beispiel #2
0
 def dispatch_background_task(self,
                              func,
                              arg,
                              uniq=None,
                              high_priority=False):
     """Submit a background task and return its handle."""
     task = Task(func,
                 arg,
                 uniq,
                 background=True,
                 high_priority=high_priority)
     taskset = Taskset([task])
     self.do_taskset(taskset)
     return task.handle
Beispiel #3
0
    def do_task(self, task):
        """Return the result of the task or raise a TaskFailed exception on failure."""

        rtn = defer.Deferred()

        def _on_fail():
            rtn.errback(self.TaskFailed("Task failed"))

        task.on_fail.append(_on_fail)
        task.on_complete.append(rtn.callback)

        taskset = Taskset([task])
        taskset.handles = {}
        
        c = self._submit_task(task)
        taskset.connections = set([c])

        def _do_cmd(*args):
            return self._command_handler(taskset, c, *args)

        with _ConnectionWrapper(c, _do_cmd):
            rtn = yield rtn

        defer.returnValue(rtn)
Beispiel #4
0
def run_worker(worker, arg, **kwargs):
    discover_workers()
    task = Task(worker, arg, **kwargs)
    if client:
        if task.background:
            taskset = Taskset([task])
            try:
                client.do_taskset(taskset)
            except GearmanConnection.ConnectionError:
                # try again, perhaps server connection was reset
                client.do_taskset(taskset)
            return task.handle
        else:
            return client.do_task(task)
    else:
        if workers.has_key(worker):
            return workers[worker](task)
        else:
            raise NotImplementedError()
Beispiel #5
0
    urls = []
    con = sqlite3.connect(db)
    for tb in CATES:
        sql = 'select * from %s limit %d' % (tb, TRAIN_URLS)
        rows = con.execute(sql)
        urls.extend(['%s\t%s' % (row[0].encode('utf-8'), tb) for row in rows])
        pass

    return urls


if __name__ == '__main__':
    urls = get_urls(URLS_DB)

    client = GearmanClient(['10.61.0.145'])
    tasks = Taskset()
    TASK_URLS_NUM = 100
    # disptribute task
    i = 0
    while i < len(urls):
        sub_urls = urls[i:i + TASK_URLS_NUM]
        workload = '\n'.join(sub_urls)
        t = Task1('crawl',
                  workload,
                  str(i),
                  timeout=TASK1_TIMEOUT,
                  retry_count=1)
        tasks.add(t)
        print "add task:%s" % t.uniq
        i += TASK_URLS_NUM
        # test
Beispiel #6
0
    urls = []
    con = sqlite3.connect(db)
    for tb in CATES:
        sql = "select * from %s limit %d" % (tb, TRAIN_URLS)
        rows = con.execute(sql)
        urls.extend(["%s\t%s" % (row[0].encode("utf-8"), tb) for row in rows])
        pass

    return urls


if __name__ == "__main__":
    urls = get_urls(URLS_DB)

    client = GearmanClient(["10.61.0.145"])
    tasks = Taskset()
    TASK_URLS_NUM = 100
    # disptribute task
    i = 0
    while i < len(urls):
        sub_urls = urls[i : i + TASK_URLS_NUM]
        workload = "\n".join(sub_urls)
        t = Task1("crawl", workload, str(i), timeout=TASK1_TIMEOUT, retry_count=1)
        tasks.add(t)
        print "add task:%s" % t.uniq
        i += TASK_URLS_NUM
        # test
        pass

    # 0.init database for return result from worker
    print "0.initialize database for results."