async def aio_resolve(subdomain_list, process_num, coroutine_num): """ 异步解析子域A记录 :param list subdomain_list: 待解析的子域列表 :param int process_num: 解析进程数 :param int coroutine_num: 每个解析进程下的协程数 :return: 解析结果 """ m = Manager() done_obj = m.Value('done', 0) # 创建一个进程间可以共享的值 loop = asyncio.get_event_loop() loop.run_in_executor(None, resolve_progress_func, done_obj, len(subdomain_list)) wrapped_resolve_func = functools.partial(do_resolve, done_obj) result_list = list() # macOS上队列大小不能超过2**15 - 1 = 32767 # https://stackoverflow.com/questions/5900985/multiprocessing-queue-maxsize-limit-is-32767 if sys.platform == 'darwin': split_subdomain_list = utils.split_list(subdomain_list, 32767) for current_subdomain_list in split_subdomain_list: async with aiomp.Pool(processes=process_num, childconcurrency=coroutine_num) as pool: result = await pool.map(wrapped_resolve_func, current_subdomain_list) result_list.extend(result) return result_list async with aiomp.Pool(processes=process_num, childconcurrency=coroutine_num) as pool: result_list = await pool.map(wrapped_resolve_func, subdomain_list) return result_list
async def main(urls,backs): all_tasks = [] for b in backs: for u in urls: all_tasks.append(u+b) for b in backup_suffix: for u in urls: try: all_tasks.append(u+'/'+u.split('//')[1].split('.')[1] + b) except: pass try: all_tasks.append(u+'/'+u.split('//')[1] + b) except: pass try: all_tasks.append(u+'/'+u.split('.', 1)[1].replace('/', '') + b) except: pass print('目标数量:{}'.format(len(all_tasks))) time.sleep(2) if len(all_tasks)>2000000: print('目标数量过于庞大,可能会导致扫描过程中内存溢出') async with aiomultiprocess.Pool() as pool: await pool.map(run,all_tasks)
async def main(self, domain, rx_queue): if not self.fuzz: # fuzz模式不探测域名是否使用泛解析 self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard( domain) tasks = self.gen_tasks(domain) logger.log('INFOR', f'正在爆破{domain}的域名') for task in tqdm.tqdm(tasks, desc='Progress', smoothing=1.0, ncols=True): async with aiomp.Pool(processes=self.process, initializer=init_worker, childconcurrency=self.coroutine) as pool: try: results = await pool.map(resolve.aiodns_query_a, task) except KeyboardInterrupt: logger.log('ALERT', '爆破终止正在退出') pool.terminate() # 关闭pool,结束工作进程,不在处理未完成的任务。 self.save_json() self.gen_result() rx_queue.put(self.results) return self.deal_results(results) self.save_json() self.gen_result() rx_queue.put(self.results)
async def get_result_from_dns(self, subhosts): res = set() async with aiomultiprocess.Pool( processes=processes, childconcurrency=childconcurrency) as pool: # 如果你想跑满CPU 修改上面的线程和协程数量即可 results = await pool.map(self.Aio_Subdomain, subhosts) for result in results: subdomain, answers = result if answers != None and subdomain != None: # res.add(subdomain) if answers != self.FakeDomain_IP: # 这里则确认不存在泛解析,可以,但是没必要 res.add(subdomain) else: try: close_old_connections() BLACKURL.objects.create(url='http://' + subdomain, ip=str(answers), title=RequestsTitle('http://' + subdomain), resons='泛解析过滤或者网址无法正常访问') except: pass res.add(self.FakeDomain_IP) return list(res)
async def main(self, domain, rx_queue): if not self.fuzz: # fuzz模式不探测域名是否使用泛解析 self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \ = detect_wildcard(domain) tasks = self.gen_tasks(domain) logger.log('INFOR', f'正在爆破{domain}的域名') # for task in tqdm.tqdm(tasks, total=len(tasks), # desc='Progress'): m = Manager() pr_queue = m.Queue() loop = asyncio.get_event_loop() loop.run_in_executor(None, progress, pr_queue, len(tasks)) wrapped_query = functools.partial(aiodns_query_a, pr_queue) async with aiomp.Pool(processes=self.process, initializer=init_worker, childconcurrency=self.coroutine) as pool: try: results = await pool.map(wrapped_query, tasks) except KeyboardInterrupt: logger.log('ALERT', '爆破终止正在退出') pool.terminate() # 关闭pool,结束工作进程,不在处理未完成的任务。 self.save_json() self.gen_result() rx_queue.put(self.results) return self.deal_results(results) self.save_json() self.gen_result() rx_queue.put(self.results)
async def main(): tasks = [] while not q.empty(): url = q.get() tasks.append(url) async with aiomultiprocess.Pool() as pool: result = await pool.map(fetch, tasks)
async def run(indicators: dict) -> None: """Accept a dict containing yara indicators and write out to the OUTPUT_DIR specified by chirp.common. :param indicators: A NamespaceDict containing parsed yara indicator files. :type indicators: dict """ if not indicators: return CONSOLE("[cyan][YARA][/cyan] Entered yara plugin.") files = [i["indicator"]["files"] for i in indicators] files = "\\**" if "\\**" in files else ", ".join(files) if files == "\\**": blame = [ i["name"] for i in indicators if i["indicator"]["files"] == "\\**" ] CONSOLE( "[cyan][YARA][/cyan] Enumerating the entire filesystem due to {}... this is going to take a while." .format(blame)) report = { indicator["name"]: build_report(indicator) for indicator in indicators } hits = 0 run_args = [] # Normalize every path, for every path try: run_args = [(a, b, indicators) for a, b in enumerate(normalize_paths(files), 1)] async with aiomp.Pool() as pool: try: async for result in pool.map(_run, tuple(run_args)): if result: report[result["namespace"]]["matches"].append( result) hits += 1 except KeyboardInterrupt: pass except IndexError: pass count = len(run_args) CONSOLE("[cyan][YARA][/cyan] Done. Processed {} files.".format(count)) CONSOLE( "[cyan][YARA][/cyan] Found {} hit(s) for yara indicators.".format( hits)) with open(os.path.join(OUTPUT_DIR, "yara.json"), "w+") as writeout: writeout.write( json.dumps( {r: report[r] for r in report if report[r]["matches"]}))
async def test_map_reduce(self): numbers = [1, 2, 3] expected = sum(numbers) * 2 async with amp.Pool() as pool: result = await pool.map_reduce(mapper, reducer, numbers) self.assertEqual(result, expected)
async def test_raise(self): result = await amp.Worker(target=raise_fn, name="test_process", initializer=do_nothing) self.assertIsInstance(result, RuntimeError) async with amp.Pool(2) as pool: with self.assertRaises(ProxyException): await pool.apply(raise_fn, args=())
async def test_pool_uvloop(self): try: import uvloop async with amp.Pool(2, loop_initializer=uvloop.new_event_loop) as pool: had_uvloop = await pool.apply(check_uvloop) self.assertTrue(had_uvloop) except ModuleNotFoundError: self.skipTest("uvloop not available")
async def get_result_from_dns(self,subhosts): res = set() async with aiomultiprocess.Pool(processes=8,childconcurrency=8) as pool: # 如果你想跑满CPU 修改上面的线程和协程数量即可 results = await pool.map(self.Aio_Subdomain, subhosts) for result in results: subdomain, answers = result if answers != None and subdomain != None: res.add(subdomain) print((subdomain,answers)) return list(res)
async def test_pool_exception_handler(self): exc_q = get_context().Queue() handler = exc_q.put_nowait async with amp.Pool(2, exception_handler=handler) as pool: with self.assertRaises(ProxyException): await pool.apply(raise_fn, args=()) exc = exc_q.get_nowait() self.assertIsInstance(exc, RuntimeError) self.assertEqual(exc.args, ("raising", ))
async def run(self): """Run the downloader with multiprocessing and multithreading.""" pool_tasks = [] async with aiomultiprocess.Pool(processes=4, maxtasksperchild=64, childconcurrency=8, queuecount=2) as pool: for call in self.calls_list: pool_tasks.append(pool.apply(self._get_call, args=[call])) for download in tqdm(asyncio.as_completed(pool_tasks), total=len(pool_tasks)): await download
async def test_initializer(self): p = amp.Process(target=sleepy, name="test_process", initializer=do_nothing) p.start() await p.join() result = 10 async with amp.Pool(2, initializer=initializer, initargs=(result, )) as pool: self.assertEqual(await pool.apply(get_dummy_constant, args=()), result)
async def main(urls, backs): # async with aiomultiprocess.Process() as process: # for u in urls: # for b in backs: # await process() all_tasks = [] for b in backs: for u in urls: all_tasks.append(u + b) print('目标数量:{}'.format(len(all_tasks))) async with aiomultiprocess.Pool() as pool: await pool.map(run, all_tasks)
async def test_pool_concurrency(self): results = [] for sleep, tasks, processes, concurrency in PERF_SETS: with Timer() as timer: async with amp.Pool(processes, childconcurrency=concurrency) as pool: await pool.map(sleepy, (sleep for _ in range(tasks))) results.append((sleep, tasks, processes, concurrency, timer.result)) print() for result in results: print(*result)
async def test_pool(self): values = list(range(10)) results = [await mapper(i) for i in values] async with amp.Pool(2, maxtasksperchild=5) as pool: self.assertEqual(pool.process_count, 2) self.assertEqual(len(pool.processes), 2) self.assertEqual(await pool.apply(mapper, (values[0],)), results[0]) self.assertEqual(await pool.map(mapper, values), results) self.assertEqual( await pool.starmap(starmapper, [values[:4], values[4:]]), [results[:4], results[4:]], )
async def test_pool_closed(self): pool = amp.Pool(2) pool.close() with self.assertRaisesRegex(RuntimeError, "pool is closed"): await pool.apply(two) with self.assertRaisesRegex(RuntimeError, "pool is closed"): await pool.map(mapper, [1, 2, 3]) with self.assertRaisesRegex(RuntimeError, "pool is closed"): await pool.starmap(starmapper, [[1, 2, 3], [1, 2, 3]]) pool.terminate()
async def test_pool_map(self): values = list(range(0, 20, 2)) expected = [k * 2 for k in values] async with amp.Pool(2) as pool: obj = pool.map(mapper, values) self.assertIsInstance(obj, amp.PoolResult) results = await obj self.assertEqual(results, expected) obj = pool.map(mapper, values) self.assertIsInstance(obj, amp.PoolResult) idx = 0 async for result in obj: self.assertEqual(result, expected[idx]) idx += 1
async def get_result_from_dns(self, subhosts): res = set() async with aiomultiprocess.Pool( processes=processes, childconcurrency=childconcurrency) as pool: # 如果你想跑满CPU 修改上面的线程和协程数量即可 results = await pool.map(self.Aio_Subdomain, subhosts) for result in results: subdomain, answers = result if answers != None and subdomain != None: if answers != self.FakeDomain_IP: # 这里则确认不存在泛解析 res.add(subdomain) else: res.add(self.FakeDomain_IP) return list(res)
async def test_spawn_method(self): self.assertEqual(amp.core.context.get_start_method(), "spawn") async def inline(x): return x with self.assertRaises(AttributeError): await amp.Worker(target=inline, args=(1, ), name="test_inline") result = await amp.Worker(target=two, name="test_global") self.assertEqual(result, 2) values = list(range(10)) results = [await mapper(i) for i in values] async with amp.Pool(2) as pool: self.assertEqual(await pool.map(mapper, values), results)
async def test_pool_starmap(self): values = list(range(0, 20, 2)) expected = [k * 2 for k in values] async with amp.Pool(2) as pool: obj = pool.starmap(starmapper, [values] * 5) self.assertIsInstance(obj, amp.PoolResult) results = await obj self.assertEqual(results, [expected] * 5) obj = pool.starmap(starmapper, [values] * 5) self.assertIsInstance(obj, amp.PoolResult) count = 0 async for results in obj: self.assertEqual(results, expected) count += 1 self.assertEqual(count, 5)
async def aio_resolve(subdomain_list, process_num, coroutine_num): """ 异步解析子域A记录 :param list subdomain_list: 待解析的子域列表 :param int process_num: 解析进程数 :param int coroutine_num: 每个解析进程下的协程数 :return: 解析结果 """ m = Manager() pr_queue = m.Queue() loop = asyncio.get_event_loop() loop.run_in_executor(None, resolve_progress, pr_queue, len(subdomain_list)) wrapped_query = functools.partial(aio_query, pr_queue) async with aiomp.Pool(processes=process_num, childconcurrency=coroutine_num) as pool: results = await pool.map(wrapped_query, subdomain_list) return results
async def run(indicators: dict) -> None: """Accept a dict containing events indicators and writes out to the OUTPUT_DIR specified by chirp.common. :param indicators: A dict containing parsed events indicator files. :type indicators: dict """ if not indicators: return hits = 0 num_logs = 0 logging.debug("Entered events plugin.") event_types = { indicator["indicator"]["event_type"] for indicator in indicators } report = { indicator["name"]: build_report(indicator) for indicator in indicators } run_args = [(event_type, indicators, report, num_logs) for event_type in event_types] async with aiomp.Pool() as pool: try: async for i in pool.map(_run, tuple(run_args)): _rep = i[0] num_logs += i[1] for k, v in _rep.items(): try: report[k]["_search_criteria"] = v["_search_criteria"] except KeyError: pass report[k]["matches"] += v["matches"] except KeyboardInterrupt: pass hits = sum(len(v["matches"]) for _, v in report.items()) logging.log(EVENTS, "Read {} logs, found {} matches.".format(num_logs, hits)) with open(os.path.join(OUTPUT_DIR, "events.json"), "w+") as writeout: writeout.write( json.dumps({r: report[r] for r in report if report[r]["matches"]}))
async def test_spawn_context(self): with self.assertRaises(ValueError): amp.set_context("foo") async def inline(x): return x amp.set_context("spawn") with self.assertRaises(AttributeError): p = amp.Worker(target=inline, args=(1, ), name="test_inline") p.start() await p.join() p = amp.Worker(target=two, name="test_global") p.start() await p.join() values = list(range(10)) results = [await mapper(i) for i in values] async with amp.Pool(2) as pool: self.assertEqual(await pool.map(mapper, values), results) self.assertEqual(p.result, 2)
async def main(urls): async with aiomultiprocess.Pool() as pool: await pool.map(run, urls)
async def main(urls, backs): all_tasks = set() for b in backs: for u in urls: # 加载普通字典 /a.rar,/2016.rar all_tasks.add(u + b) # http://www.langzi.fun/a.rar try: # 加载自定义目录+普通字典 all_tasks.add(u + '/' + u.split('//')[1].split('.')[1] + b) # http://www.langzi.fun/langzi/a.rar except: pass for m in backup_dir: all_tasks.add(u + '/' + m + b) # 加载目录 # http://www.langzi.fun/www/a.rar for b in first_back_: # 动态加载自定义的后缀名,比如/a.zip,/a.rar,/a.iso for u in urls: all_tasks.add(u + b) for b in backup_suffix: # 自定义根据域名生成检测网址 for u in urls: try: all_tasks.add(u + '/' + u.split('//')[1].split('.')[1] + b) # http://www.langzi.fun/langzi.rar except: pass for m in backup_dir: try: # 加载目录 all_tasks.add(u + '/' + m + '/' + u.split('//')[1].split('.')[1] + b) # http://www.langzi.fun/www/langzi.rar except: pass try: all_tasks.add(u + '/' + u.split('//')[1] + b) # http://www.langzi.fun/www.langzi.fun.rar except: pass try: all_tasks.add(u + '/' + u.split('.', 1)[1].replace('/', '') + b) # http://www.langzi.fun/langzi.rar except: pass try: all_tasks.add(u + '/' + u.split('//')[1].split('.')[1] + '/' + u.split('//')[1].split('.')[1] + b) # http://www.langzi.fun/langzi/langzi.rar except: pass all_tasks = list(all_tasks) print('目标数量:{}'.format(len(all_tasks))) time.sleep(2) if len(all_tasks) > 3000000: print('目标数量过于庞大,可能会导致扫描过程中内存溢出导致蓝屏死机') print('开始扫描.....') async with aiomultiprocess.Pool() as pool: await pool.map(run, all_tasks)
async def main(urls): async with aiomultiprocess.Pool() as pool: result = await pool.map(run, urls) return result
async def main(self,urls): async with aiomultiprocess.Pool(processes=8,childconcurrency=8) as pool: result = await pool.map(self.check_url_alive,urls) return [x for x in result if x is not None]
async def main(): tasks = range(800, 1001) async with aiomultiprocess.Pool() as pool: result = await pool.map(tq,tasks) print(result)