class ThreadPool(object): def __init__(self, count=0): # 设置Pool运行状态 self.running = True from os import cpu_count # 用到的时候导入对应模块即可 # 默认是CPU核数,且至少有一个线程 if count <= 0: count = cpu_count() or 1 # 设置线程数 self.queue = Queue(count) # 启动对应个数的线程 for _ in range(count): Task(self.queue) # 不能在这直接启动,会阻塞Pool的 def apply_async(self, func, args=(), kws={}): if self.running: # 执行任务 self.queue.put((func, args, kws)) def close(self): # 不再运行加入任务 self.running = False def join(self): # 等待任务执行完退出 self.queue.join()
def send_emails(modeladmin, request, queryset): messages = Queue() for user in queryset: process = Process(target=send_email, args=(user, messages)) process.start() messages.get().send() process.join()
class Build(object): def __init__(self): pool = Pool(processes=2) self.graph = getGraph() files = findFiles(opts) self.progressQueue = Queue() reporter = Process(target=ProgressReport, args=(self.progressQueue, len(files))) reporter.start() result = pool.map(self.cacheFile, enumerate(files), chunksize=5) self.progressQueue.put('END') log.info("finished, %s results", len(result)) reporter.join() def cacheFile(self, (i, filename)): log.debug("cacheFile %s" % filename) try: uri = photoUri(filename) m = MediaResource(self.graph, uri) # video extension but not isVideo? where do we correct that? m.cacheAll() except Exception: traceback.print_exc() self.progressQueue.put("%s" % filename)
def run(self, videosupport=False, debug=False, cpulimit=False): '''start the application!''' self.tque = Queue() #(maxsize=120) self.framequeue = Queue() #(maxsize=120) self.cvcommandqueue = Queue() self.videosupport = videosupport self.cvthread = None if self.videosupport: self.cvthread = threading.Thread(target=nnlib.cvworker, args=(self.tque, self.cvcommandqueue, cpulimit)) self.cvthread.daemon = True self.cvthread.start() self.tthread = threading.Thread(target=nnlib.tfworker, args=(self.tque, self.framequeue)) self.tthread.daemon = True self.tthread.start() #''' self.updatethread = threading.Thread(target=updatethread, args=(self, )) self.updatethread.daemon = True self.updatethread.start() #''' self.debug = debug if self.debug: self.root.after(100, self.set_image, 'demo5.jpg') self.root.mainloop()
def __init__(self, db_name, proc_count, site_base_url, fUseCache=True, fCacheSearchPages=True, fUseCookies=False, timeout=secHTTP_WAIT_TIMEOUT, search_proc_count=2, proxies=None): self.proxies = proxies self.queue = Queue() self.fSaveSearchPages = fCacheSearchPages self.site_base_url = site_base_url self.pool = Pool(processes=proc_count) self.search_queue = Queue() self.url_extract_pool = Pool(processes=search_proc_count) URL_Fetcher.__init__(self, db_name, fUseCache, fUseCookies, timeout=timeout, proxies=proxies)
def main(): '''load video, process frames, display to user''' tque = Queue() #(maxsize=120) framequeue = Queue() #(maxsize=120) cthread = threading.Thread(target=cvworker, args=(tque, )) cthread.daemon = True cthread.start() tthread = threading.Thread(target=tfworker, args=(tque, framequeue)) tthread.daemon = True #terminate testloop when user closes window tthread.start() start = time.time() frame = 0 videoend = False while True: cvw = cv2.waitKey(1) if cvw & 0xFF == ord('q'): break if not videoend: print('got', frame, time.time()) frame += 1 print('frame:', frame) f = framequeue.get() if type(f) == type(None): videoend = True pass #whats this do else: #time.sleep(1/30) #limit to realtime cv2.imshow('frame', f) print('new took:', time.time() - start) cv2.destroyAllWindows()
def _get_system_information_threaded(host): system_information_functions = [ collect_win_application_stats, collect_win_bios_stats, collect_win_disk_stats, collect_win_local_account_stats, collect_win_local_group_stats, collect_win_mem_stats, collect_win_network_stats, collect_os_stats, collect_win_processes_stats, collect_win_cpu_stats, collect_win_services_stats ] system_information = {} queue = Queue() list_of_processes = [] for hardware in system_information_functions: process = _Process(target=hardware, args=( host, 1, queue, )) list_of_processes.append(process) process.start() for process in list_of_processes: process.join() system_information.update(queue.get()) return system_information
def __init__(self, feeder, fifo_path, end_nl_q=None, daemon=True, *args): os.mkfifo(fifo_path) super().__init__(daemon=daemon) self.feeder = feeder self.fifo_path = fifo_path self.end_nl_q = Queue() if end_nl_q is None else end_nl_q self._exception = None self._want_join = threading.Event()
def parallel_apply(func, iterable, workers, max_queue_size, callback=None, dummy=False): """多进程或多线程地将func应用到iterable的每个元素中。 注意这个apply是异步且无序的,也就是说依次输入a,b,c,但是 输出可能是func(c), func(a), func(b)。 参数: dummy: False是多进程/线性,True则是多线程/线性; callback: 处理单个输出的回调函数; """ if dummy: from multiprocessing.dummy import Pool, Queue else: from multiprocessing import Pool, Queue in_queue, out_queue = Queue(max_queue_size), Queue() # 启动多进程/线程 pool = Pool(workers, worker_step, (func, in_queue, out_queue)) if callback is None: results = [] # 后处理函数 def process_out_queue(): out_count = 0 for _ in range(out_queue.qsize()): d = out_queue.get() out_count += 1 if callback is None: results.append(d) else: callback(d) return out_count # 存入数据,取出结果 in_count, out_count = 0, 0 for d in iterable: in_count += 1 while True: try: in_queue.put(d, block=False) break except six.moves.queue.Full: out_count += process_out_queue() if in_count % max_queue_size == 0: out_count += process_out_queue() while out_count != in_count: out_count += process_out_queue() pool.terminate() if callback is None: return results
class seed(Process): """ Consumer thread that takes fuzzable requests from a Queue that's populated by the crawl plugins and identified vulnerabilities by performing various requests. """ def __init__(self, w3af_core): """ :param w3af_core: The w3af core that we'll use for status reporting """ super(seed, self).__init__() self.name = "Seed" self._w3af_core = w3af_core # See documentation in the property below self._out_queue = Queue() def get_result(self, timeout=0.5): return self._out_queue.get_nowait() def has_pending_work(self): return self._out_queue.qsize() != 0 def join(self): return def terminate(self): return def seed_output_queue(self, target_urls): """ Create the first fuzzable request objects based on the targets and put them in the output Queue. This will start the whole discovery process, since plugins are going to consume from that Queue and then put their results in it again in order to continue discovering. """ # We only want to scan pages that are in current scope in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain() for url in target_urls: try: # # GET the initial target URLs in order to save them # in a list and use them as our bootstrap URLs # response = self._w3af_core.uri_opener.GET(url, cache=True) except (w3afMustStopOnUrlError, w3afException, w3afMustStopException), w3: om.out.error("The target URL: %s is unreachable." % url) om.out.error("Error description: %s" % w3) except Exception, e: om.out.error("The target URL: %s is unreachable " "because of an unhandled exception." % url) om.out.error('Error description: "%s". See debug ' "output for more information." % e) om.out.error("Traceback for this error: %s" % traceback.format_exc()) else:
def __init__(self, model_dir, path_group_dict: Dict[str, int]): self.feed = Queue() self.mid = Queue() self.out = Queue() t = MtCNN(self.feed, self.mid) t.start() t1 = EmbeddingCmp(self.feed, self.mid, self.out, path_group_dict, model_dir) t1.start()
def __init__(self, w3af_core): """ :param w3af_core: The w3af core that we'll use for status reporting """ super(seed, self).__init__(name='%sController' % self.get_name()) self._w3af_core = w3af_core # See documentation in the property below self._out_queue = Queue()
class Actor(object): def __init__(self): # Actor内部的消息缓存队列 self.__mailbox = Queue() def send(self, msg): self.__mailbox.put(msg) def recv(self): return self.__mailbox.get()
def __init__(self, w3af_core): ''' :param w3af_core: The w3af core that we'll use for status reporting ''' super(seed, self).__init__() self.name = 'Seed' self._w3af_core = w3af_core # See documentation in the property below self._out_queue = Queue()
def __init__(self): self.queue = list() self.queue_index = -1 self.play_queue_order = list() self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"]) self.current_play_mode = self.play_modes.NORMAL self.now_playing_stream = None # Create multiprocess queues self.task_queue = Queue() self.done_queue = Queue() # Workers self.workers = list()
def __init__(self, base_path, callback=None): if not os.path.isdir(base_path): raise IOError("Base path not found: " + base_path) self.base_path = base_path self.unsearched = Manager().Queue() self.dirpath_queue = Queue() self.cpu_count = multiprocessing.cpu_count() self.pool = Pool(self.cpu_count) self.first_level_dirs = "" self.callback = callback
def _add_doi(metadata, identifier, citekey): """Add an entry from a DOI.""" info_messages = [] with StatusMessage('Querying DOI metadata...') as message: if metadata.doi_exists(identifier): raise ZoiaAddException(f'DOI {identifier} already exists.') # Query Semantic Scholar to get the corresponding arxiv ID (if there is # one) in a separate thread. arxiv_queue = ThreadQueue() arxiv_process = ThreadProcess( target=lambda q, x: q.put(requests.get(x)), args=( arxiv_queue, f'https://api.semanticscholar.org/v1/paper/{identifier}', ), ) arxiv_process.start() doi_metadata = _get_doi_metadata(identifier) metadatum = zoia.backend.metadata.Metadatum.from_dict(doi_metadata) if citekey is None: citekey = zoia.parse.citekey.create_citekey(metadata, metadatum) paper_dir = os.path.join(metadata.config.library_root, citekey) os.mkdir(paper_dir) message.update( 'Querying Semantic Scholar for corresponding arXiv ID...') arxiv_metadata_response = arxiv_queue.get() arxiv_process.join() arxiv_metadata = json.loads(arxiv_metadata_response.text) if (arxiv_id := arxiv_metadata.get('arxivId')) is not None: doi_metadata['arxiv_id'] = arxiv_id message.update('Downloading PDF from arXiv...') pdf_response = requests.get( f'https://arxiv.org/pdf/{arxiv_id}.pdf') if pdf_response.status_code == 200: with open(os.path.join(paper_dir, 'document.pdf'), 'wb') as fp: fp.write(pdf_response.content) doi_metadata['pdf_md5'] = hashlib.md5( pdf_response.content).hexdigest() else: info_messages.append('Was unable to fetch a PDF') metadata[citekey] = doi_metadata
def run(self): in_queue, out_queue = Queue(), Queue() for i in self.a: in_queue.put(i) def f(in_queue, out_queue): while not in_queue.empty(): time.sleep(1) out_queue.put(in_queue.get()+1) pool = Pool(4, f, (in_queue, out_queue)) self.b = [] while len(self.b) < len(self.a): if not out_queue.empty(): self.b.append(out_queue.get()) pool.terminate()
def main(q=None): """ Try to detect relics on the screen and show if they can be detected in a window. For some weird reason, Qt and tesseract cannot run in the same thread. Doing so leads to a crash and I'm unable to figure out the cause. We workaround this by creating a thread for Qt and putting all ocr result into a queue. :param q: A queue for communication with the Qt thread """ if q is not None: app = QApplication(sys.argv) widget = Widget(q) sys.exit(app.exec_()) q = Queue(1) p = Process(target=main, args=(q, )) p.start() tessdata_dir = 'tessdata/' with TesserocrPool(tessdata_dir, 'Roboto', psm=PSM.SINGLE_BLOCK, oem=OEM.LSTM_ONLY) as pool, mss.mss() as sct: s = Screenshots(sct) while p.is_alive(): begin = time.time() image_input = next(s) end = time.time() delta = end - begin print(f'screenshot took {delta}s') try: ocr_data = do_ocr(pool, image_input) except: ocr_data = None if ocr_data is None: ocr_data = itertools.repeat(('ocrerror', ) * 4, 20) try: q.put(tuple(ocr_data), block=True, timeout=0.5) except QueueFullException: if not p.is_alive(): break except ValueError as e: pass except AssertionError as e: pass p.join()
def run_tasks(tasks, context=None, nb_threads=1, watchdog=None): got_keyboard_interrupt = False watchdogs = [ lambda _: _KEYBOARD_INTERRUPT_ERROR_MESSAGE if got_keyboard_interrupt else None ] if watchdog: watchdogs.append(watchdog) for task in tasks: check_task_dependencies(task) remaining_tasks = list(tasks) completed_tasks = list() pool = Pool(nb_threads) completed_tasks_queue = Queue() try: schedule_tasks_to_be_run( pop_runnable_tasks(remaining_tasks, completed_tasks, nb_threads), watchdogs, context, pool, completed_tasks_queue) while len(completed_tasks) != len(tasks): # wait for one task to complete completed_task = completed_tasks_queue.get() completed_tasks.append(completed_task) # schedule tasks to be run waiting for task success or simple completion tasks_to_be_run = pop_runnable_tasks(remaining_tasks, completed_tasks, nb_threads) schedule_tasks_to_be_run(tasks_to_be_run, watchdogs, context, pool, completed_tasks_queue) except KeyboardInterrupt: got_keyboard_interrupt = True skip_all_tasks(tasks, remaining_tasks, completed_tasks, context, pool, completed_tasks_queue, _KEYBOARD_INTERRUPT_ERROR_MESSAGE) finally: pool.close() exceptions = [ task.result.stacktrace for task in tasks if isinstance(task.result, TaskResultException) ] if exceptions: raise TasksExecutionFailure("Caught exceptions:\n%s" % "\n".join(exceptions))
def __init__(self, api_key=API_KEY): self.queue = list() self.queue_index = -1 self.play_queue_order = list() self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"]) self.current_play_mode = self.play_modes.NORMAL self.now_playing_stream = None # Create multiprocess queues self.task_queue = Queue() self.done_queue = Queue() # Workers self.workers = list() self.api_key = api_key if api_key != "" else API_KEY pafy.set_api_key(self.api_key)
class Recon: def __init__(self, model_dir, path_group_dict: Dict[str, int]): self.feed = Queue() self.mid = Queue() self.out = Queue() t = MtCNN(self.feed, self.mid) t.start() t1 = EmbeddingCmp(self.feed, self.mid, self.out, path_group_dict, model_dir) t1.start() def face_check(self, _im: np.ndarray, group: int): self.feed.put((_im, group)) return self.out.get()
def __init__(self, count=0): # 设置Pool运行状态 self.running = True from os import cpu_count # 用到的时候导入对应模块即可 # 默认是CPU核数,且至少有一个线程 if count <= 0: count = cpu_count() or 1 # 设置线程数 self.queue = Queue(count) # 启动对应个数的线程 for _ in range(count): Task(self.queue) # 不能在这直接启动,会阻塞Pool的
class BaseActor(object): def __init__(self): """queue:Actor内部的邮箱队列""" self.__mailbox = Queue() def recv(self): """Actor接受消息""" msg = self.__mailbox.get() if msg is ActorExit: # 抛出异常(模版方法会处理) raise ActorExit return msg def send(self, msg): """Actor发送消息""" self.__mailbox.put(msg) def close(self): """发送结束标识""" self.send(ActorExit) def start(self): self.__terminated_event = Event() # 为Join服务 t = threading.Thread(target=self.__templet) t.setDaemon(True) # 设置为守护线程 t.start() def __templet(self): """模版方法(run会被子类重写)""" try: self.run() # 执行Run代码 except ActorExit: pass # 防止线程挂掉 finally: # 设置Event标识 self.__terminated_event.set() def join(self): # Event在set之后便结束等待 self.__terminated_event.wait() def run(self): """ 由子类实现即可,eg: while True: msg = self.recv() print(msg) """ pass
class FIFOFeeder(threading.Thread): def __init__(self, feeder, fifo_path, end_nl_q=None, daemon=True, *args): os.mkfifo(fifo_path) super().__init__(daemon=daemon) self.feeder = feeder self.fifo_path = fifo_path self.end_nl_q = Queue() if end_nl_q is None else end_nl_q self._exception = None self._want_join = threading.Event() def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_value, exc_tb): self.join() def run(self): try: # Try to open the FIFO nonblocking, so we can periodically check # if the main thread wants us to wind down. If it does, there's no # more need for the FIFO, so stop the thread. while True: try: fd = os.open(self.fifo_path, os.O_WRONLY | os.O_NONBLOCK) except OSError as error: if error.errno != errno.ENXIO: raise elif self._want_join.is_set(): return else: break # Now clear the fd's nonblocking flag to let writes block normally. fcntl.fcntl(fd, fcntl.F_SETFL, 0) with open(fd, 'wb') as fifo: # The queue works around a unified diff limitation: if there's # no newlines in both don't make it a difference end_nl = self.feeder(fifo) self.end_nl_q.put(end_nl) except Exception as error: self._exception = error def join(self): self._want_join.set() super().join() if self._exception is not None: raise self._exception
def main(): queue = Queue() # 开启生产消费者线程任务 t_list = [ threading.Thread(target=func, args=(queue, )) for func in (producer, consumer) ] # 启动两个线程 for t in t_list: # 设置后台线程,就算是死循环当主线程退出的时候也会退出的 t.setDaemon(True) # 进程是daemon属性,t.daemon=True t.start() # 等待所有任务完成 queue.join() # 你可以把这句话注释掉看输出 print(f"当前队列未完成的数量:{queue.unfinished_tasks}")
def __init__(self): pool = Pool(processes=2) self.graph = getGraph() files = findFiles(opts) self.progressQueue = Queue() reporter = Process(target=ProgressReport, args=(self.progressQueue, len(files))) reporter.start() result = pool.map(self.cacheFile, enumerate(files), chunksize=5) self.progressQueue.put('END') log.info("finished, %s results", len(result)) reporter.join()
def __init__(self): '''初始化''' # 创建队列 self._queue = Queue() # 是否触发时间 self._active = False # 线程池 self._thread = [Process(target=self._run) for _ in range(5)] self._workers_n = 5 # 执行 self._handlers = {}
def set_symbols_mode(self, symbols, mode): """ 货币对采取多进程或多线程模式,使用队列 :param symbols: :param mode: process多进程,thread多线程 :return: """ if mode == "process": manager = mp.Manager() queue = manager.Queue() else: queue = Queue() for symbol in symbols: queue.put(symbol) self.symbol_queue = queue
def main(): queue = Queue() pool = ThreadPool() pool.apply_async(consumer, args=(queue, )) pool.apply_async(producer, args=(queue, )) pool.close() pool.join()
def run(self): in_queue, out_queue = Queue(), Queue() for i in self.a: in_queue.put(i) def f(in_queue, out_queue): while not in_queue.empty(): time.sleep(1) out_queue.put(in_queue.get() + 1) pool = Pool(4, f, (in_queue, out_queue)) self.b = [] while len(self.b) < len(self.a): if not out_queue.empty(): self.b.append(out_queue.get()) pool.terminate()
class FrontServer(): def __init__(self): self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.port = settings.port self.loginQueue = ThreadQue() def start(self): self.loginSocket.bind(('localhost', self.port)) self.loginSocket.listen(200) for x in range(10): thread = threading.Thread(target=loginWorker, args=(self.loginQueue, )) thread.daemon = True thread.start() requestQueue = Queue() Game.instance().requestQueue = requestQueue updater = ServerUpdater() receiver = RequestReader() cookieMover = CookieMover() updateThread = threading.Thread(target=updater.run) updateThread.daemon = True receiverThread = threading.Thread(target=receiver.run, args=(requestQueue, )) receiverThread.daemon = True moverThread = threading.Thread(target=cookieMover.run) moverThread.daemon = True updateThread.start() receiverThread.start() moverThread.start() try: print('Server started listening') while True: clientSocket, clientAddress = self.loginSocket.accept() print('Connection made with front server') self.loginQueue.put(clientSocket) except (KeyboardInterrupt, SystemExit, socket.error): self.loginSocket.shutdown(socket.SHUT_RDWR) self.loginSocket.close() sys.exit()
class FrontServer(): def __init__(self): self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.port = settings.port self.loginQueue = ThreadQue() def start(self): self.loginSocket.bind(('localhost',self.port)) self.loginSocket.listen(200) for x in range(10): thread = threading.Thread(target=loginWorker, args=(self.loginQueue,)) thread.daemon = True thread.start() requestQueue = Queue() Game.instance().requestQueue = requestQueue updater = ServerUpdater() receiver = RequestReader() cookieMover = CookieMover() updateThread = threading.Thread(target=updater.run) updateThread.daemon = True receiverThread = threading.Thread(target=receiver.run, args=(requestQueue,)) receiverThread.daemon = True moverThread = threading.Thread(target=cookieMover.run) moverThread.daemon = True updateThread.start() receiverThread.start() moverThread.start() try: print('Server started listening') while True: clientSocket, clientAddress = self.loginSocket.accept() print('Connection made with front server') self.loginQueue.put(clientSocket) except (KeyboardInterrupt, SystemExit, socket.error): self.loginSocket.shutdown(socket.SHUT_RDWR) self.loginSocket.close() sys.exit()
def stat_files(): all_files = [] for root, dirs, files in os.walk('/home/gzguoyubo/mf/tw2/res/entities/custom_type'): ignore = False for ig_path in ignore_paths: if ig_path in root: ignore = True if ignore: continue for fname in files: if not fname.endswith('.py'): continue abs_file_path = join(root, fname) all_files.append(abs_file_path) file_sections = [] file_total_nums = len(all_files) for i in xrange(P_NUM): start = i * file_total_nums / P_NUM stop = start + file_total_nums / P_NUM if i == P_NUM - 1: stop = -1 file_sections.append(all_files[start : stop]) res_queue = Queue() processes = [] for section in file_sections: p = Process(target=stat_file, args=(section, res_queue)) p.start() processes.append(p) for p in processes: p.join() total_stats = defaultdict(int) while not res_queue.empty(): stat = res_queue.get() for author, cnt in stat.iteritems(): total_stats[author] += cnt print total_stats
class ExThread(Thread): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__status_queue = Queue() def run(self, *args, **kwargs): try: super().run(*args, **kwargs) except Exception as ex: #except_type, except_class, tb = sys.exc_info() self.__status_queue.put(ex) self.__status_queue.put(None) def wait_for_exc_info(self): return self.__status_queue.get() def join(self): ex = self.wait_for_exc_info() if ex is None: return else: raise ex
class Server(): def __init__(self, host, port, basedir): self.host = host self.port = port self.basedir = self.__formatBaseDir(basedir) self.socket = s.socket(s.AF_INET, s.SOCK_STREAM) self.q = ThreadQue() def start(self): for x in range(100): thread = threading.Thread(target=responder, args=(self.q,)) thread.daemon = True thread.start() try: self.socket.bind((self.host, self.port)) self.socket.listen(500) print('Server starting on port '+str(self.port)+' from base directory '+self.basedir) while True: clientSocket, clientAddress = self.socket.accept() self.q.put((clientSocket, self.basedir)) except (KeyboardInterrupt, s.error): try: self.socket.shutdown(s.SHUT_RDWR) self.socket.close() print('Server socket closed') sys.exit() except s.error: print('Server port busy. Try another port.') sys.exit() def __formatBaseDir(self, basedir): #Trim trailing slash for convenience if basedir[len(basedir) - 1] is os.path.sep: basedir = basedir[:-1] return basedir
def get_stats(): print 'Fetching NBA player stats...' stats_outfile = RUNDAY+'_nba_stats.csv' csvout = open(stats_outfile, 'wb') NUM_THREADS = 8 in_queue = Queue() out_queue = Queue() queue_players(in_queue) while not in_queue.empty(): jobs = [] for i in range(NUM_THREADS): if not in_queue.empty(): thread = Process(target=get_stats_helper, args=(in_queue, out_queue)) jobs.append(thread) thread.start() for thread in jobs: thread.join() while not out_queue.empty(): player = out_queue.get() del player['SUCCESS'] try: name = player['NAME'] except KeyError as e: continue player['TIME'] = RUNDAY fieldnames = [ 'TIME', 'NAME', 'JERSEY', 'SPORT', 'TEAM', 'POSITION', 'PTS', 'REB', 'AST', 'URL' ] csvwriter = csv.DictWriter(csvout, delimiter='|', fieldnames=fieldnames) csvwriter.writerow(player) csvout.close() print 'Finished fetching NBA player stats.' print 'Ouput saved in %s' % stats_outfile
def __init__(self, host, port, basedir): self.host = host self.port = port self.basedir = self.__formatBaseDir(basedir) self.socket = s.socket(s.AF_INET, s.SOCK_STREAM) self.q = ThreadQue()
def __init__(self): self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.port = settings.port self.loginQueue = ThreadQue()
# encoding=utf-8 ''' @author: liangzai pip install multiprocessing ''' from multiprocessing.dummy import Pool, Queue import threading import time # 队列 tasks = Queue() for i in range(9): tasks.put(str(i)*2) def main(name): while True: time.sleep(1) if name.empty(): print("name is over.") break else: thread = threading.current_thread().getName() print("线程:%s 打印:%s" % (thread, name.get())) # 启动四个线程 pool = Pool(4, main, (tasks,)) while True: time.sleep(5)
class seed(Process): """ Consumer thread that takes fuzzable requests from a Queue that's populated by the crawl plugins and identified vulnerabilities by performing various requests. """ def __init__(self, w3af_core): """ :param w3af_core: The w3af core that we'll use for status reporting """ super(seed, self).__init__(name='%sController' % self.get_name()) self._w3af_core = w3af_core # See documentation in the property below self._out_queue = Queue() def get_name(self): return 'Seed' def get_result(self, timeout=0.5): return self._out_queue.get_nowait() def has_pending_work(self): return self._out_queue.qsize() != 0 def join(self): return @property def out_queue(self): # This output queue can contain one of the following: # * POISON_PILL # * (plugin_name, fuzzable_request, AsyncResult) # * An ExceptionData instance return self._out_queue def terminate(self): while True: try: self._out_queue.get_nowait() except Empty: break else: self._out_queue.task_done() om.out.debug('No more tasks in Seed consumer output queue.') def seed_output_queue(self, target_urls): """ Create the first fuzzable request objects based on the targets and put them in the output Queue. This will start the whole discovery process, since plugins are going to consume from that Queue and then put their results in it again in order to continue discovering. """ # We only want to scan pages that are in current scope in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain() for url in target_urls: try: # # GET the initial target URLs in order to save them # in a list and use them as our bootstrap URLs # response = self._w3af_core.uri_opener.GET(url, cache=True) except ScanMustStopException, w3: om.out.error('The target server is unreachable. Stopping.') raise w3 except HTTPRequestException, hre: msg = 'The target URL: "%s" is unreachable. Exception: "%s".' om.out.error(msg % (url, hre)) except Exception, e: msg = ('The target URL: "%s" is unreachable because of an' ' unhandled exception. Error description: "%s". See' ' debug output for more information.\n' 'Traceback for this error:\n%s') om.out.error(msg % (url, e, traceback.format_exc()))
class seed(Process): """ Consumer thread that takes fuzzable requests from a Queue that's populated by the crawl plugins and identified vulnerabilities by performing various requests. """ def __init__(self, w3af_core): """ :param w3af_core: The w3af core that we'll use for status reporting """ super(seed, self).__init__(name='SeedController') self.name = 'Seed' self._w3af_core = w3af_core # See documentation in the property below self._out_queue = Queue() def get_result(self, timeout=0.5): return self._out_queue.get_nowait() def has_pending_work(self): return self._out_queue.qsize() != 0 def join(self): return def terminate(self): return def seed_output_queue(self, target_urls): """ Create the first fuzzable request objects based on the targets and put them in the output Queue. This will start the whole discovery process, since plugins are going to consume from that Queue and then put their results in it again in order to continue discovering. """ # We only want to scan pages that are in current scope in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain() for url in target_urls: # batman-fix already done verify, no need below codes # try: # # # # GET the initial target URLs in order to save them # # in a list and use them as our bootstrap URLs # # # response = self._w3af_core.uri_opener.GET(url, cache=True) # except ScanMustStopException, w3: # om.out.error('The target server is unreachable. Stopping.') # raise w3 # except HTTPRequestException, hre: # msg = 'The target URL: "%s" is unreachable. Exception: "%s".' # om.out.error(msg % (url, hre)) # except Exception, e: # msg = 'The target URL: "%s" is unreachable because of an' \ # ' unhandled exception. Error description: "%s". See' \ # ' debug output for more information.\n' \ # 'Traceback for this error:\n%s' # om.out.error(msg % (url, e, traceback.format_exc())) # else: # _seed = FuzzableRequest(response.get_uri()) _seed = FuzzableRequest(url) if in_scope(_seed): self._out_queue.put((None, None, _seed)) # Update the set that lives in the KB kb.kb.add_fuzzable_request(_seed) self._out_queue.put(POISON_PILL)
class tizyoutubeproxy(object): """A class that accesses YouTube, retrieves stream URLs and creates and manages a playback queue. """ def __init__(self): self.queue = list() self.queue_index = -1 self.play_queue_order = list() self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"]) self.current_play_mode = self.play_modes.NORMAL self.now_playing_stream = None # Create multiprocess queues self.task_queue = Queue() self.done_queue = Queue() # Workers self.workers = list() def set_play_mode(self, mode): """ Set the playback mode. :param mode: current valid values are "NORMAL" and "SHUFFLE" """ self.current_play_mode = getattr(self.play_modes, mode) self.__update_play_queue_order() def enqueue_audio_stream(self, arg): """Add the audio stream of a YouTube video to the playback queue. :param arg: a search string """ logging.info('arg : %s', arg) try: yt_video = pafy.new(arg) yt_audio = yt_video.getbestaudio(preftype="webm") if not yt_audio: raise ValueError(str("No WebM audio stream for : %s" % arg)) yt_info = VideoInfo(ytid=arg, title=yt_audio.title) self.add_to_playback_queue(audio=yt_audio, video=yt_video, info=yt_info) self.__update_play_queue_order() except ValueError: raise ValueError(str("Video not found : %s" % arg)) def enqueue_audio_playlist(self, arg): """Add all audio streams in a YouTube playlist to the playback queue. :param arg: a YouTube playlist id """ logging.info('arg : %s', arg) try: count = len(self.queue) playlist = pafy.get_playlist2(arg) if len(playlist) > 0: for yt_video in playlist: self.add_to_playback_queue(video=yt_video, \ info=VideoInfo(ytid=yt_video.videoid, \ title=yt_video.title)) if count == len(self.queue): raise ValueError self.__update_play_queue_order() except ValueError: raise ValueError(str("Playlist not found : %s" % arg)) def enqueue_audio_search(self, arg): """Search YouTube and add the audio streams to the playback queue. :param arg: a search string """ logging.info('arg : %s', arg) try: query = generate_search_query(arg) wdata = pafy.call_gdata('search', query) wdata2 = wdata count = 0 while True: for track_info in get_tracks_from_json(wdata2): self.add_to_playback_queue(info=track_info) count += 1 if count > 100: break if not wdata2.get('nextPageToken'): break query['pageToken'] = wdata2['nextPageToken'] wdata2 = pafy.call_gdata('search', query) self.__update_play_queue_order() except ValueError: raise ValueError(str("Could not find any mixes : %s" % arg)) def enqueue_audio_mix(self, arg, feelinglucky=True): """Obtain a YouTube mix associated to a given video id or url and add all audio streams in the mix playlist to the playback queue. :param arg: a YouTube video id :param feelinglucky: If True, it will perform another YouTube search to find alternatives if the original mix cannot be found. """ logging.info('arg : %s', arg) yt_video = None try: count = len(self.queue) yt_video = pafy.new(arg) playlist = yt_video.mix if len(playlist) > 0: for yt_video in playlist: video_id = yt_video.videoid video_title = yt_video.title yt_info = VideoInfo(ytid=video_id, title=video_title) self.add_to_playback_queue(video=yt_video, info=yt_info) if count == len(self.queue): raise ValueError self.__update_play_queue_order() except IndexError: if not feelinglucky: raise ValueError else: print_wrn("[YouTube] Could not find a mix for '{0}'. "\ "Searching YouTube instead. Feeling lucky?." \ .format(arg.encode('utf-8'))) if yt_video.title: self.enqueue_audio_search(yt_video.title) else: self.enqueue_audio_stream(arg) def enqueue_audio_mix_search(self, arg): """Obtain a YouTube mix associated to a given textual search and add all the audio streams in the mix playlist to the playback queue. :param arg: a search string """ logging.info('arg : %s', arg) try: query = generate_search_query(arg) wdata = pafy.call_gdata('search', query) wdata2 = wdata count = len(self.queue) for track_info in get_tracks_from_json(wdata2): if track_info and track_info.ytid: try: self.enqueue_audio_mix(track_info.ytid, feelinglucky=False) break except ValueError: logging.info('Could not find a mix. Trying another video') if count == len(self.queue): raise ValueError except ValueError: raise ValueError(str("Could not find any mixes : %s" % arg)) def enqueue_audio_channel_uploads(self, arg): """Add all audio streams in a YouTube channel to the playback queue. :param arg: a YouTube channel url """ logging.info('arg : %s', arg) try: count = len(self.queue) channel = pafy.get_channel(arg) if channel: for yt_video in channel.uploads: self.add_to_playback_queue(video=yt_video, \ info=VideoInfo(ytid=yt_video.videoid, \ title=yt_video.title)) if count == len(self.queue): raise ValueError self.__update_play_queue_order() except ValueError: raise ValueError(str("Channel not found : %s" % arg)) def enqueue_audio_channel_playlist(self, channel_name, playlist_name): """Search a playlist within a channel and if found, adds all the audio streams to the playback queue. :param arg: a YouTube playlist id """ logging.info('args : %s - %s', channel_name, playlist_name) try: count = len(self.queue) channel = pafy.get_channel(channel_name) if channel: pl_dict = dict() pl_titles = list() pl_name = '' playlist = None for pl in channel.playlists: print_nfo("[YouTube] [Playlist] '{0}'." \ .format(to_ascii(pl.title))) if fuzz.partial_ratio(playlist_name, pl.title) > 50: pl_dict[pl.title] = pl pl_titles.append(pl.title) if len(pl_titles) > 1: pl_name = process.extractOne(playlist_name, pl_titles)[0] playlist = pl_dict[pl_name] elif len(pl_titles) == 1: pl_name = pl_titles[0] playlist = pl_dict[pl_name] if pl_name: if pl_name.lower() != playlist_name.lower(): print_wrn("[YouTube] Playlist '{0}' not found. " \ "Playing '{1}' instead." \ .format(to_ascii(playlist_name), \ to_ascii(pl_name))) for yt_video in playlist: self.add_to_playback_queue(video=yt_video, \ info=VideoInfo(ytid=yt_video.videoid, \ title=yt_video.title)) if count == len(self.queue): raise ValueError self.__update_play_queue_order() except ValueError: raise ValueError(str("Channel not found : %s" % channel_name)) def current_audio_stream_title(self): """ Retrieve the current stream's title. """ stream = self.now_playing_stream title = '' if stream: title = to_ascii(stream['a'].title).encode("utf-8") return title def current_audio_stream_author(self): """ Retrieve the current stream's author. """ stream = self.now_playing_stream author = '' if stream: author = to_ascii(stream['v'].author).encode("utf-8") return author def current_audio_stream_file_size(self): """ Retrieve the current stream's file size. """ stream = self.now_playing_stream size = 0 if stream: size = stream['a'].get_filesize() return size def current_audio_stream_duration(self): """ Retrieve the current stream's duration. """ stream = self.now_playing_stream duration = '' if stream: duration = to_ascii(stream['v'].duration).encode("utf-8") return duration def current_audio_stream_bitrate(self): """ Retrieve the current stream's bitrate. """ stream = self.now_playing_stream bitrate = '' if stream: bitrate = stream['a'].bitrate return bitrate def current_audio_stream_view_count(self): """ Retrieve the current stream's view count. """ stream = self.now_playing_stream viewcount = 0 if stream: viewcount = stream['v'].viewcount return viewcount def current_audio_stream_description(self): """ Retrieve the current stream's description. """ stream = self.now_playing_stream description = '' if stream: description = to_ascii(stream['v'].description).encode("utf-8") return description def current_audio_stream_file_extension(self): """ Retrieve the current stream's file extension. """ stream = self.now_playing_stream file_extension = '' if stream: file_extension = to_ascii(stream['a'].extension).encode("utf-8") return file_extension def current_audio_stream_video_id(self): """ Retrieve the current stream's video id. """ stream = self.now_playing_stream video_id = '' if stream: video_id = to_ascii(stream['i'].ytid).encode("utf-8") return video_id def current_audio_stream_published(self): """ Retrieve the current stream's upload date and time. """ stream = self.now_playing_stream if stream: published = to_ascii(stream['v'].published).encode("utf-8") return published def current_audio_stream_queue_index_and_queue_length(self): """ Retrieve index in the queue (starting from 1) of the current stream and the length of the playback queue. """ return self.queue_index + 1, len(self.queue) def clear_queue(self): """ Clears the playback queue. """ self.queue = list() self.queue_index = -1 def remove_current_url(self): """Remove the currently active url from the playback queue. """ logging.info("") if len(self.queue) and self.queue_index: stream = self.queue[self.queue_index] print_nfo("[YouTube] [Stream] '{0}' removed." \ .format(to_ascii(stream['i'].title).encode("utf-8"))) del self.queue[self.queue_index] self.queue_index -= 1 if self.queue_index < 0: self.queue_index = 0 self.__update_play_queue_order() def next_url(self): """ Retrieve the url of the next stream in the playback queue. """ logging.info("") try: if len(self.queue): self.queue_index += 1 if (self.queue_index < len(self.queue)) \ and (self.queue_index >= 0): next_stream = self.queue[self.play_queue_order \ [self.queue_index]] return self.__retrieve_stream_url(next_stream, self.queue_index).rstrip() else: self.queue_index = -1 return self.next_url() else: return '' except (KeyError, AttributeError): # TODO: We don't remove this for now # del self.queue[self.queue_index] logging.info("KeyError, or AttributeError exception") return self.next_url() except (IOError): # Remove this video del self.queue[self.queue_index] logging.info("IOError exception") return self.next_url() def prev_url(self): """ Retrieve the url of the previous stream in the playback queue. """ logging.info("") try: if len(self.queue): self.queue_index -= 1 if (self.queue_index < len(self.queue)) \ and (self.queue_index >= 0): prev_stream = self.queue[self.play_queue_order \ [self.queue_index]] return self.__retrieve_stream_url(prev_stream, self.queue_index).rstrip() else: self.queue_index = len(self.queue) return self.prev_url() else: return '' except (KeyError, AttributeError): # TODO: We don't remove this for now # del self.queue[self.queue_index] logging.info("exception") return self.prev_url() except (IOError): # Remove this video del self.queue[self.queue_index] logging.info("IOError exception") return self.next_url() def __update_play_queue_order(self): """ Update the queue playback order. A sequential order is applied if the current play mode is "NORMAL" or a random order if current play mode is "SHUFFLE" """ total_streams = len(self.queue) if total_streams: if not len(self.play_queue_order): # Create a sequential play order, if empty self.play_queue_order = range(total_streams) if self.current_play_mode == self.play_modes.SHUFFLE: random.shuffle(self.play_queue_order) print_nfo("[YouTube] [Streams in queue] '{0}'." \ .format(total_streams)) def __retrieve_stream_url(self, stream, queue_index): """ Retrieve a stream url """ try: if not len(self.workers): for _ in range(WORKER_PROCESSES): proc = Process(target=obtain_stream, \ args=(self.task_queue, \ self.done_queue)).start() self.workers.append(proc) while not self.done_queue.empty(): stream = self.done_queue.get() self.queue[stream['q']] = stream stream = self.queue[queue_index] if not stream.get('v') or not stream.get('a'): logging.info("ytid : %s", stream['i'].ytid) video = stream.get('v') if not video: video = pafy.new(stream['i'].ytid) audio = video.getbestaudio(preftype="webm") if not audio: logging.info("no suitable audio found") raise AttributeError() stream.update({'a': audio, 'v': video}) # streams = stream.get('v').audiostreams[::-1] # pprint.pprint(streams) # dump_stream_info(streams) self.now_playing_stream = stream return stream['a'].url.encode("utf-8") except AttributeError: logging.info("Could not retrieve the stream url!") raise def add_to_playback_queue(self, audio=None, video=None, info=None): """ Add to the playback queue. """ if audio: print_nfo("[YouTube] [Stream] '{0}' [{1}]." \ .format(to_ascii(audio.title).encode("utf-8"), \ to_ascii(audio.extension))) if info: print_nfo("[YouTube] [Stream] '{0}'." \ .format(to_ascii(info.title).encode("utf-8"))) queue_index = len(self.queue) self.task_queue.put(dict(a=audio, v=video, i=info, q=queue_index)) self.queue.append( dict(a=audio, v=video, i=info, q=queue_index))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__status_queue = Queue()