def f(args): def x(*y): print(y) grp = GreenPool() list(grp.starmap(x, [[1, 2, 3] for x in range(1000)])) grp.waitall()
def _get_patches(self, project): commits = self.repository_rpc.get_commits(project) chunks = utilities.chunk(commits, size=round(len(commits) * 0.01)) pool = GreenPool(os.cpu_count()) arguments = [(project, c, self.repository_rpc) for c in chunks] patches = list() for _patches in pool.starmap(_get_patches, arguments): patches.extend(_patches) return PatchSchema(many=True).load(patches)
def main(myid, queue, concurrency, delay=5.0, duration=DURATION): counter = 0 created = list() results = LightQueue(concurrency * 10) pool = GreenPool(concurrency) api = AccountClient({'namespace': NS}, pool_maxsize=concurrency+1) now = start = checkpoint = time.time() pool.starmap(create_loop, [(api, 'buck-%d-%d' % (myid, n), results) for n in range(concurrency)]) while now - start < duration: try: res = results.get(timeout=delay) created.append(res) counter += 1 except Empty: pass if now - checkpoint > delay: print("Proc %d: %d updates in %fs, %f updates per second." % ( myid, counter, now - checkpoint, counter / (now - checkpoint))) counter = 0 checkpoint = now now = time.time() for coro in pool.coroutines_running: coro.kill() while not results.empty(): created.append(results.get(block=False)) end = time.time() rate = len(created) / (end - start) print("Proc %d: end. %d updates in %fs, %f updates per second." % ( myid, len(created), end - start, rate)) time.sleep(2) print("Proc %d: cleaning..." % myid) del_req = {'dtime': time.time()} # Do not delete twice (or an exception is raised) uniq_ct = set(created) for _ in pool.starmap(api.container_update, [(ACCOUNT, n, del_req) for n in uniq_ct]): pass pool.waitall() queue.put(rate) return 0
def collect(self, project, **options): logger.debug(project) project = ProjectSchema().load(self.project_rpc.get(project)) if project.language.lower() not in self.config['KEYWORDS']: raise LanguageNotSupported(f'{project.language} not supported') keywords = self.config['KEYWORDS'].get(project.language.lower()) keywrd = Keywrd(keywords=keywords) commits = self.repository_rpc.get_commits(project.name) chunks = utilities.chunk(commits, size=round(len(commits) * 0.01)) pool = GreenPool(os.cpu_count()) arguments = [(project, c, self.repository_rpc) for c in chunks] keyword = list() for patches in pool.starmap(_get_patches, arguments): patches = PatchSchema(many=True).load(patches) keyword.extend(keywrd.get(patches)) return KeywordSchema(many=True).dump(keyword)