def f(args):
def x(*y):
print(y)
grp = GreenPool()
list(grp.starmap(x, [[1, 2, 3] for x in range(1000)]))
grp.waitall()
def _get_patches(self, project):
commits = self.repository_rpc.get_commits(project)
chunks = utilities.chunk(commits, size=round(len(commits) * 0.01))
pool = GreenPool(os.cpu_count())
arguments = [(project, c, self.repository_rpc) for c in chunks]
patches = list()
for _patches in pool.starmap(_get_patches, arguments):
patches.extend(_patches)
return PatchSchema(many=True).load(patches)
def main(myid, queue, concurrency, delay=5.0, duration=DURATION):
counter = 0
created = list()
results = LightQueue(concurrency * 10)
pool = GreenPool(concurrency)
api = AccountClient({'namespace': NS}, pool_maxsize=concurrency+1)
now = start = checkpoint = time.time()
pool.starmap(create_loop, [(api, 'buck-%d-%d' % (myid, n), results)
for n in range(concurrency)])
while now - start < duration:
try:
res = results.get(timeout=delay)
created.append(res)
counter += 1
except Empty:
pass
if now - checkpoint > delay:
print("Proc %d: %d updates in %fs, %f updates per second." % (
myid, counter, now - checkpoint,
counter / (now - checkpoint)))
counter = 0
checkpoint = now
now = time.time()
for coro in pool.coroutines_running:
coro.kill()
while not results.empty():
created.append(results.get(block=False))
end = time.time()
rate = len(created) / (end - start)
print("Proc %d: end. %d updates in %fs, %f updates per second." % (
myid, len(created), end - start, rate))
time.sleep(2)
print("Proc %d: cleaning..." % myid)
del_req = {'dtime': time.time()}
# Do not delete twice (or an exception is raised)
uniq_ct = set(created)
for _ in pool.starmap(api.container_update,
[(ACCOUNT, n, del_req) for n in uniq_ct]):
pass
pool.waitall()
queue.put(rate)
return 0
def collect(self, project, **options):
logger.debug(project)
project = ProjectSchema().load(self.project_rpc.get(project))
if project.language.lower() not in self.config['KEYWORDS']:
raise LanguageNotSupported(f'{project.language} not supported')
keywords = self.config['KEYWORDS'].get(project.language.lower())
keywrd = Keywrd(keywords=keywords)
commits = self.repository_rpc.get_commits(project.name)
chunks = utilities.chunk(commits, size=round(len(commits) * 0.01))
pool = GreenPool(os.cpu_count())
arguments = [(project, c, self.repository_rpc) for c in chunks]
keyword = list()
for patches in pool.starmap(_get_patches, arguments):
patches = PatchSchema(many=True).load(patches)
keyword.extend(keywrd.get(patches))
return KeywordSchema(many=True).dump(keyword)
