def run(self, params): def inner(i): return connections.redis.get("test") if params["subpool_size"]: subpool_map(params["subpool_size"], inner, range(0, params["subpool_size"] * 5)) else: inner(0)
def run(self, params): self.config = get_current_config() concurrency = int(params.get("concurrency", 5)) groups = self.fetch_worker_group_definitions() if len(groups) == 0: log.error("No worker group definition yet. Can't orchestrate!") return subpool_map(concurrency, self.orchestrate, groups)
def run(self, params): get_clients = lambda: [c for c in connections.redis.client_list() if c.get("cmd") != "client"] def inner(i): print "Greenlet #%s, %s clients so far" % (id(gevent.getcurrent()), len(get_clients())) return connections.redis.get("test") if params["subpool_size"]: subpool_map(params["subpool_size"], inner, range(0, params["subpool_size"] * 5)) else: inner(0)
def run(self, params): self.job = get_current_job() if params.get("imap"): return subpool_map(params["pool_size"], self.inner, params["inner_params"]) else: return list(subpool_imap(params["pool_size"], self.inner, params["inner_params"]))
def run(self, params): def get_clients(): return [ c for c in connections.redis.client_list() if c.get("cmd") != "client" ] def inner(i): print("Greenlet #%s, %s clients so far" % (id(gevent.getcurrent()), len(get_clients()))) return connections.redis.get("test") if params["subpool_size"]: subpool_map(params["subpool_size"], inner, list(range(0, params["subpool_size"] * 5))) else: inner(0)
def send_tasks(path, params_list, queue=None, sync=False, batch_size=1000, subpool_size=None): if sync: def subpool_inner_func(params): try: return run_task(path, params) except Exception as exc: trace = traceback.format_exc() log.error("Error in send_tasks subpool: %s \n%s" % (exc, trace)) raise if subpool_size is None: return [run_task(path, params) for params in params_list] else: return subpool_map(int(subpool_size), subpool_inner_func, params_list) return queue_jobs(path, params_list, queue=queue, batch_size=batch_size)
def run(self, params): self.job = get_current_job() return subpool_map(params["pool_size"], self.inner, params["inner_params"])