class ThreadPool(object):
def __init__(self, count=0):
# 设置Pool运行状态
self.running = True
from os import cpu_count # 用到的时候导入对应模块即可
# 默认是CPU核数，且至少有一个线程
if count <= 0:
count = cpu_count() or 1
# 设置线程数
self.queue = Queue(count)
# 启动对应个数的线程
for _ in range(count):
Task(self.queue) # 不能在这直接启动，会阻塞Pool的
def apply_async(self, func, args=(), kws={}):
if self.running:
# 执行任务
self.queue.put((func, args, kws))
def close(self):
# 不再运行加入任务
self.running = False
def join(self):
# 等待任务执行完退出
self.queue.join()
def send_emails(modeladmin, request, queryset):
messages = Queue()
for user in queryset:
process = Process(target=send_email, args=(user, messages))
process.start()
messages.get().send()
process.join()
class Build(object):
def __init__(self):
pool = Pool(processes=2)
self.graph = getGraph()
files = findFiles(opts)
self.progressQueue = Queue()
reporter = Process(target=ProgressReport,
args=(self.progressQueue, len(files)))
reporter.start()
result = pool.map(self.cacheFile, enumerate(files), chunksize=5)
self.progressQueue.put('END')
log.info("finished, %s results", len(result))
reporter.join()
def cacheFile(self, (i, filename)):
log.debug("cacheFile %s" % filename)
try:
uri = photoUri(filename)
m = MediaResource(self.graph, uri)
# video extension but not isVideo? where do we correct that?
m.cacheAll()
except Exception:
traceback.print_exc()
self.progressQueue.put("%s" % filename)
def run(self, videosupport=False, debug=False, cpulimit=False):
'''start the application!'''
self.tque = Queue() #(maxsize=120)
self.framequeue = Queue() #(maxsize=120)
self.cvcommandqueue = Queue()
self.videosupport = videosupport
self.cvthread = None
if self.videosupport:
self.cvthread = threading.Thread(target=nnlib.cvworker,
args=(self.tque,
self.cvcommandqueue,
cpulimit))
self.cvthread.daemon = True
self.cvthread.start()
self.tthread = threading.Thread(target=nnlib.tfworker,
args=(self.tque, self.framequeue))
self.tthread.daemon = True
self.tthread.start()
#'''
self.updatethread = threading.Thread(target=updatethread,
args=(self, ))
self.updatethread.daemon = True
self.updatethread.start()
#'''
self.debug = debug
if self.debug:
self.root.after(100, self.set_image, 'demo5.jpg')
self.root.mainloop()
class Build(object):
def __init__(self):
pool = Pool(processes=2)
self.graph = getGraph()
files = findFiles(opts)
self.progressQueue = Queue()
reporter = Process(target=ProgressReport,
args=(self.progressQueue, len(files)))
reporter.start()
result = pool.map(self.cacheFile, enumerate(files), chunksize=5)
self.progressQueue.put('END')
log.info("finished, %s results", len(result))
reporter.join()
def cacheFile(self, (i, filename)):
log.debug("cacheFile %s" % filename)
try:
uri = photoUri(filename)
m = MediaResource(self.graph, uri)
# video extension but not isVideo? where do we correct that?
m.cacheAll()
except Exception:
traceback.print_exc()
self.progressQueue.put("%s" % filename)
def __init__(self,
db_name,
proc_count,
site_base_url,
fUseCache=True,
fCacheSearchPages=True,
fUseCookies=False,
timeout=secHTTP_WAIT_TIMEOUT,
search_proc_count=2,
proxies=None):
self.proxies = proxies
self.queue = Queue()
self.fSaveSearchPages = fCacheSearchPages
self.site_base_url = site_base_url
self.pool = Pool(processes=proc_count)
self.search_queue = Queue()
self.url_extract_pool = Pool(processes=search_proc_count)
URL_Fetcher.__init__(self,
db_name,
fUseCache,
fUseCookies,
timeout=timeout,
proxies=proxies)
def main():
'''load video, process frames, display to user'''
tque = Queue() #(maxsize=120)
framequeue = Queue() #(maxsize=120)
cthread = threading.Thread(target=cvworker, args=(tque, ))
cthread.daemon = True
cthread.start()
tthread = threading.Thread(target=tfworker, args=(tque, framequeue))
tthread.daemon = True #terminate testloop when user closes window
tthread.start()
start = time.time()
frame = 0
videoend = False
while True:
cvw = cv2.waitKey(1)
if cvw & 0xFF == ord('q'): break
if not videoend:
print('got', frame, time.time())
frame += 1
print('frame:', frame)
f = framequeue.get()
if type(f) == type(None):
videoend = True
pass #whats this do
else:
#time.sleep(1/30) #limit to realtime
cv2.imshow('frame', f)
print('new took:', time.time() - start)
cv2.destroyAllWindows()
def _get_system_information_threaded(host):
system_information_functions = [
collect_win_application_stats, collect_win_bios_stats,
collect_win_disk_stats, collect_win_local_account_stats,
collect_win_local_group_stats, collect_win_mem_stats,
collect_win_network_stats, collect_os_stats,
collect_win_processes_stats, collect_win_cpu_stats,
collect_win_services_stats
]
system_information = {}
queue = Queue()
list_of_processes = []
for hardware in system_information_functions:
process = _Process(target=hardware, args=(
host,
1,
queue,
))
list_of_processes.append(process)
process.start()
for process in list_of_processes:
process.join()
system_information.update(queue.get())
return system_information
def __init__(self, feeder, fifo_path, end_nl_q=None, daemon=True, *args):
os.mkfifo(fifo_path)
super().__init__(daemon=daemon)
self.feeder = feeder
self.fifo_path = fifo_path
self.end_nl_q = Queue() if end_nl_q is None else end_nl_q
self._exception = None
self._want_join = threading.Event()
def parallel_apply(func,
iterable,
workers,
max_queue_size,
callback=None,
dummy=False):
"""多进程或多线程地将func应用到iterable的每个元素中。
注意这个apply是异步且无序的，也就是说依次输入a,b,c，但是
输出可能是func(c), func(a), func(b)。
参数：
dummy: False是多进程/线性，True则是多线程/线性；
callback: 处理单个输出的回调函数；
"""
if dummy:
from multiprocessing.dummy import Pool, Queue
else:
from multiprocessing import Pool, Queue
in_queue, out_queue = Queue(max_queue_size), Queue()
# 启动多进程/线程
pool = Pool(workers, worker_step, (func, in_queue, out_queue))
if callback is None:
results = []
# 后处理函数
def process_out_queue():
out_count = 0
for _ in range(out_queue.qsize()):
d = out_queue.get()
out_count += 1
if callback is None:
results.append(d)
else:
callback(d)
return out_count
# 存入数据，取出结果
in_count, out_count = 0, 0
for d in iterable:
in_count += 1
while True:
try:
in_queue.put(d, block=False)
break
except six.moves.queue.Full:
out_count += process_out_queue()
if in_count % max_queue_size == 0:
out_count += process_out_queue()
while out_count != in_count:
out_count += process_out_queue()
pool.terminate()
if callback is None:
return results
class seed(Process):
"""
Consumer thread that takes fuzzable requests from a Queue that's populated
by the crawl plugins and identified vulnerabilities by performing various
requests.
"""
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__()
self.name = "Seed"
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def get_result(self, timeout=0.5):
return self._out_queue.get_nowait()
def has_pending_work(self):
return self._out_queue.qsize() != 0
def join(self):
return
def terminate(self):
return
def seed_output_queue(self, target_urls):
"""
Create the first fuzzable request objects based on the targets and put
them in the output Queue.
This will start the whole discovery process, since plugins are going
to consume from that Queue and then put their results in it again in
order to continue discovering.
"""
# We only want to scan pages that are in current scope
in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain()
for url in target_urls:
try:
#
# GET the initial target URLs in order to save them
# in a list and use them as our bootstrap URLs
#
response = self._w3af_core.uri_opener.GET(url, cache=True)
except (w3afMustStopOnUrlError, w3afException, w3afMustStopException), w3:
om.out.error("The target URL: %s is unreachable." % url)
om.out.error("Error description: %s" % w3)
except Exception, e:
om.out.error("The target URL: %s is unreachable " "because of an unhandled exception." % url)
om.out.error('Error description: "%s". See debug ' "output for more information." % e)
om.out.error("Traceback for this error: %s" % traceback.format_exc())
else:
def __init__(self, model_dir, path_group_dict: Dict[str, int]):
self.feed = Queue()
self.mid = Queue()
self.out = Queue()
t = MtCNN(self.feed, self.mid)
t.start()
t1 = EmbeddingCmp(self.feed, self.mid, self.out, path_group_dict,
model_dir)
t1.start()
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__(name='%sController' % self.get_name())
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
class Actor(object):
def __init__(self):
# Actor内部的消息缓存队列
self.__mailbox = Queue()
def send(self, msg):
self.__mailbox.put(msg)
def recv(self):
return self.__mailbox.get()
def __init__(self, w3af_core):
'''
:param w3af_core: The w3af core that we'll use for status reporting
'''
super(seed, self).__init__()
self.name = 'Seed'
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def __init__(self):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
def __init__(self):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
def __init__(self, base_path, callback=None):
if not os.path.isdir(base_path):
raise IOError("Base path not found: " + base_path)
self.base_path = base_path
self.unsearched = Manager().Queue()
self.dirpath_queue = Queue()
self.cpu_count = multiprocessing.cpu_count()
self.pool = Pool(self.cpu_count)
self.first_level_dirs = ""
self.callback = callback
def _add_doi(metadata, identifier, citekey):
"""Add an entry from a DOI."""
info_messages = []
with StatusMessage('Querying DOI metadata...') as message:
if metadata.doi_exists(identifier):
raise ZoiaAddException(f'DOI {identifier} already exists.')
# Query Semantic Scholar to get the corresponding arxiv ID (if there is
# one) in a separate thread.
arxiv_queue = ThreadQueue()
arxiv_process = ThreadProcess(
target=lambda q, x: q.put(requests.get(x)),
args=(
arxiv_queue,
f'https://api.semanticscholar.org/v1/paper/{identifier}',
),
)
arxiv_process.start()
doi_metadata = _get_doi_metadata(identifier)
metadatum = zoia.backend.metadata.Metadatum.from_dict(doi_metadata)
if citekey is None:
citekey = zoia.parse.citekey.create_citekey(metadata, metadatum)
paper_dir = os.path.join(metadata.config.library_root, citekey)
os.mkdir(paper_dir)
message.update(
'Querying Semantic Scholar for corresponding arXiv ID...')
arxiv_metadata_response = arxiv_queue.get()
arxiv_process.join()
arxiv_metadata = json.loads(arxiv_metadata_response.text)
if (arxiv_id := arxiv_metadata.get('arxivId')) is not None:
doi_metadata['arxiv_id'] = arxiv_id
message.update('Downloading PDF from arXiv...')
pdf_response = requests.get(
f'https://arxiv.org/pdf/{arxiv_id}.pdf')
if pdf_response.status_code == 200:
with open(os.path.join(paper_dir, 'document.pdf'), 'wb') as fp:
fp.write(pdf_response.content)
doi_metadata['pdf_md5'] = hashlib.md5(
pdf_response.content).hexdigest()
else:
info_messages.append('Was unable to fetch a PDF')
metadata[citekey] = doi_metadata
def run(self):
in_queue, out_queue = Queue(), Queue()
for i in self.a:
in_queue.put(i)
def f(in_queue, out_queue):
while not in_queue.empty():
time.sleep(1)
out_queue.put(in_queue.get()+1)
pool = Pool(4, f, (in_queue, out_queue))
self.b = []
while len(self.b) < len(self.a):
if not out_queue.empty():
self.b.append(out_queue.get())
pool.terminate()
def main(q=None):
"""
Try to detect relics on the screen and show if they can be detected in a window.
For some weird reason, Qt and tesseract cannot run in the same thread.
Doing so leads to a crash and I'm unable to figure out the cause.
We workaround this by creating a thread for Qt and putting all ocr result into a queue.
:param q: A queue for communication with the Qt thread
"""
if q is not None:
app = QApplication(sys.argv)
widget = Widget(q)
sys.exit(app.exec_())
q = Queue(1)
p = Process(target=main, args=(q, ))
p.start()
tessdata_dir = 'tessdata/'
with TesserocrPool(tessdata_dir,
'Roboto',
psm=PSM.SINGLE_BLOCK,
oem=OEM.LSTM_ONLY) as pool, mss.mss() as sct:
s = Screenshots(sct)
while p.is_alive():
begin = time.time()
image_input = next(s)
end = time.time()
delta = end - begin
print(f'screenshot took {delta}s')
try:
ocr_data = do_ocr(pool, image_input)
except:
ocr_data = None
if ocr_data is None:
ocr_data = itertools.repeat(('ocrerror', ) * 4, 20)
try:
q.put(tuple(ocr_data), block=True, timeout=0.5)
except QueueFullException:
if not p.is_alive():
break
except ValueError as e:
pass
except AssertionError as e:
pass
p.join()
def run_tasks(tasks, context=None, nb_threads=1, watchdog=None):
got_keyboard_interrupt = False
watchdogs = [
lambda _: _KEYBOARD_INTERRUPT_ERROR_MESSAGE
if got_keyboard_interrupt else None
]
if watchdog:
watchdogs.append(watchdog)
for task in tasks:
check_task_dependencies(task)
remaining_tasks = list(tasks)
completed_tasks = list()
pool = Pool(nb_threads)
completed_tasks_queue = Queue()
try:
schedule_tasks_to_be_run(
pop_runnable_tasks(remaining_tasks, completed_tasks, nb_threads),
watchdogs, context, pool, completed_tasks_queue)
while len(completed_tasks) != len(tasks):
# wait for one task to complete
completed_task = completed_tasks_queue.get()
completed_tasks.append(completed_task)
# schedule tasks to be run waiting for task success or simple completion
tasks_to_be_run = pop_runnable_tasks(remaining_tasks,
completed_tasks, nb_threads)
schedule_tasks_to_be_run(tasks_to_be_run, watchdogs, context, pool,
completed_tasks_queue)
except KeyboardInterrupt:
got_keyboard_interrupt = True
skip_all_tasks(tasks, remaining_tasks, completed_tasks, context, pool,
completed_tasks_queue,
_KEYBOARD_INTERRUPT_ERROR_MESSAGE)
finally:
pool.close()
exceptions = [
task.result.stacktrace for task in tasks
if isinstance(task.result, TaskResultException)
]
if exceptions:
raise TasksExecutionFailure("Caught exceptions:\n%s" %
"\n".join(exceptions))
def __init__(self, api_key=API_KEY):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
self.api_key = api_key if api_key != "" else API_KEY
pafy.set_api_key(self.api_key)
class Recon:
def __init__(self, model_dir, path_group_dict: Dict[str, int]):
self.feed = Queue()
self.mid = Queue()
self.out = Queue()
t = MtCNN(self.feed, self.mid)
t.start()
t1 = EmbeddingCmp(self.feed, self.mid, self.out, path_group_dict,
model_dir)
t1.start()
def face_check(self, _im: np.ndarray, group: int):
self.feed.put((_im, group))
return self.out.get()
def __init__(self, count=0):
# 设置Pool运行状态
self.running = True
from os import cpu_count # 用到的时候导入对应模块即可
# 默认是CPU核数，且至少有一个线程
if count <= 0:
count = cpu_count() or 1
# 设置线程数
self.queue = Queue(count)
# 启动对应个数的线程
for _ in range(count):
Task(self.queue) # 不能在这直接启动，会阻塞Pool的
class BaseActor(object):
def __init__(self):
"""queue：Actor内部的邮箱队列"""
self.__mailbox = Queue()
def recv(self):
"""Actor接受消息"""
msg = self.__mailbox.get()
if msg is ActorExit:
# 抛出异常（模版方法会处理）
raise ActorExit
return msg
def send(self, msg):
"""Actor发送消息"""
self.__mailbox.put(msg)
def close(self):
"""发送结束标识"""
self.send(ActorExit)
def start(self):
self.__terminated_event = Event() # 为Join服务
t = threading.Thread(target=self.__templet)
t.setDaemon(True) # 设置为守护线程
t.start()
def __templet(self):
"""模版方法（run会被子类重写）"""
try:
self.run() # 执行Run代码
except ActorExit:
pass # 防止线程挂掉
finally:
# 设置Event标识
self.__terminated_event.set()
def join(self):
# Event在set之后便结束等待
self.__terminated_event.wait()
def run(self):
"""
由子类实现即可，eg：
while True:
msg = self.recv()
print(msg)
"""
pass
class FIFOFeeder(threading.Thread):
def __init__(self, feeder, fifo_path, end_nl_q=None, daemon=True, *args):
os.mkfifo(fifo_path)
super().__init__(daemon=daemon)
self.feeder = feeder
self.fifo_path = fifo_path
self.end_nl_q = Queue() if end_nl_q is None else end_nl_q
self._exception = None
self._want_join = threading.Event()
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_value, exc_tb):
self.join()
def run(self):
try:
# Try to open the FIFO nonblocking, so we can periodically check
# if the main thread wants us to wind down. If it does, there's no
# more need for the FIFO, so stop the thread.
while True:
try:
fd = os.open(self.fifo_path, os.O_WRONLY | os.O_NONBLOCK)
except OSError as error:
if error.errno != errno.ENXIO:
raise
elif self._want_join.is_set():
return
else:
break
# Now clear the fd's nonblocking flag to let writes block normally.
fcntl.fcntl(fd, fcntl.F_SETFL, 0)
with open(fd, 'wb') as fifo:
# The queue works around a unified diff limitation: if there's
# no newlines in both don't make it a difference
end_nl = self.feeder(fifo)
self.end_nl_q.put(end_nl)
except Exception as error:
self._exception = error
def join(self):
self._want_join.set()
super().join()
if self._exception is not None:
raise self._exception
def main():
queue = Queue()
# 开启生产消费者线程任务
t_list = [
threading.Thread(target=func, args=(queue, ))
for func in (producer, consumer)
]
# 启动两个线程
for t in t_list:
# 设置后台线程，就算是死循环当主线程退出的时候也会退出的
t.setDaemon(True) # 进程是daemon属性，t.daemon=True
t.start()
# 等待所有任务完成
queue.join() # 你可以把这句话注释掉看输出
print(f"当前队列未完成的数量：{queue.unfinished_tasks}")
def __init__(self):
pool = Pool(processes=2)
self.graph = getGraph()
files = findFiles(opts)
self.progressQueue = Queue()
reporter = Process(target=ProgressReport,
args=(self.progressQueue, len(files)))
reporter.start()
result = pool.map(self.cacheFile, enumerate(files), chunksize=5)
self.progressQueue.put('END')
log.info("finished, %s results", len(result))
reporter.join()
def __init__(self):
'''初始化'''
# 创建队列
self._queue = Queue()
# 是否触发时间
self._active = False
# 线程池
self._thread = [Process(target=self._run) for _ in range(5)]
self._workers_n = 5
# 执行
self._handlers = {}
def set_symbols_mode(self, symbols, mode):
"""
货币对采取多进程或多线程模式，使用队列
:param symbols:
:param mode: process多进程,thread多线程
:return:
"""
if mode == "process":
manager = mp.Manager()
queue = manager.Queue()
else:
queue = Queue()
for symbol in symbols:
queue.put(symbol)
self.symbol_queue = queue
def main():
queue = Queue()
pool = ThreadPool()
pool.apply_async(consumer, args=(queue, ))
pool.apply_async(producer, args=(queue, ))
pool.close()
pool.join()
def run(self):
in_queue, out_queue = Queue(), Queue()
for i in self.a:
in_queue.put(i)
def f(in_queue, out_queue):
while not in_queue.empty():
time.sleep(1)
out_queue.put(in_queue.get() + 1)
pool = Pool(4, f, (in_queue, out_queue))
self.b = []
while len(self.b) < len(self.a):
if not out_queue.empty():
self.b.append(out_queue.get())
pool.terminate()
class FrontServer():
def __init__(self):
self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.port = settings.port
self.loginQueue = ThreadQue()
def start(self):
self.loginSocket.bind(('localhost', self.port))
self.loginSocket.listen(200)
for x in range(10):
thread = threading.Thread(target=loginWorker,
args=(self.loginQueue, ))
thread.daemon = True
thread.start()
requestQueue = Queue()
Game.instance().requestQueue = requestQueue
updater = ServerUpdater()
receiver = RequestReader()
cookieMover = CookieMover()
updateThread = threading.Thread(target=updater.run)
updateThread.daemon = True
receiverThread = threading.Thread(target=receiver.run,
args=(requestQueue, ))
receiverThread.daemon = True
moverThread = threading.Thread(target=cookieMover.run)
moverThread.daemon = True
updateThread.start()
receiverThread.start()
moverThread.start()
try:
print('Server started listening')
while True:
clientSocket, clientAddress = self.loginSocket.accept()
print('Connection made with front server')
self.loginQueue.put(clientSocket)
except (KeyboardInterrupt, SystemExit, socket.error):
self.loginSocket.shutdown(socket.SHUT_RDWR)
self.loginSocket.close()
sys.exit()
class FrontServer():
def __init__(self):
self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.port = settings.port
self.loginQueue = ThreadQue()
def start(self):
self.loginSocket.bind(('localhost',self.port))
self.loginSocket.listen(200)
for x in range(10):
thread = threading.Thread(target=loginWorker, args=(self.loginQueue,))
thread.daemon = True
thread.start()
requestQueue = Queue()
Game.instance().requestQueue = requestQueue
updater = ServerUpdater()
receiver = RequestReader()
cookieMover = CookieMover()
updateThread = threading.Thread(target=updater.run)
updateThread.daemon = True
receiverThread = threading.Thread(target=receiver.run, args=(requestQueue,))
receiverThread.daemon = True
moverThread = threading.Thread(target=cookieMover.run)
moverThread.daemon = True
updateThread.start()
receiverThread.start()
moverThread.start()
try:
print('Server started listening')
while True:
clientSocket, clientAddress = self.loginSocket.accept()
print('Connection made with front server')
self.loginQueue.put(clientSocket)
except (KeyboardInterrupt, SystemExit, socket.error):
self.loginSocket.shutdown(socket.SHUT_RDWR)
self.loginSocket.close()
sys.exit()
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__(name='%sController' % self.get_name())
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def stat_files():
all_files = []
for root, dirs, files in os.walk('/home/gzguoyubo/mf/tw2/res/entities/custom_type'):
ignore = False
for ig_path in ignore_paths:
if ig_path in root:
ignore = True
if ignore:
continue
for fname in files:
if not fname.endswith('.py'):
continue
abs_file_path = join(root, fname)
all_files.append(abs_file_path)
file_sections = []
file_total_nums = len(all_files)
for i in xrange(P_NUM):
start = i * file_total_nums / P_NUM
stop = start + file_total_nums / P_NUM
if i == P_NUM - 1:
stop = -1
file_sections.append(all_files[start : stop])
res_queue = Queue()
processes = []
for section in file_sections:
p = Process(target=stat_file, args=(section, res_queue))
p.start()
processes.append(p)
for p in processes:
p.join()
total_stats = defaultdict(int)
while not res_queue.empty():
stat = res_queue.get()
for author, cnt in stat.iteritems():
total_stats[author] += cnt
print total_stats
class ExThread(Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__status_queue = Queue()
def run(self, *args, **kwargs):
try:
super().run(*args, **kwargs)
except Exception as ex:
#except_type, except_class, tb = sys.exc_info()
self.__status_queue.put(ex)
self.__status_queue.put(None)
def wait_for_exc_info(self):
return self.__status_queue.get()
def join(self):
ex = self.wait_for_exc_info()
if ex is None:
return
else:
raise ex
class Server():
def __init__(self, host, port, basedir):
self.host = host
self.port = port
self.basedir = self.__formatBaseDir(basedir)
self.socket = s.socket(s.AF_INET, s.SOCK_STREAM)
self.q = ThreadQue()
def start(self):
for x in range(100):
thread = threading.Thread(target=responder, args=(self.q,))
thread.daemon = True
thread.start()
try:
self.socket.bind((self.host, self.port))
self.socket.listen(500)
print('Server starting on port '+str(self.port)+' from base directory '+self.basedir)
while True:
clientSocket, clientAddress = self.socket.accept()
self.q.put((clientSocket, self.basedir))
except (KeyboardInterrupt, s.error):
try:
self.socket.shutdown(s.SHUT_RDWR)
self.socket.close()
print('Server socket closed')
sys.exit()
except s.error:
print('Server port busy. Try another port.')
sys.exit()
def __formatBaseDir(self, basedir):
#Trim trailing slash for convenience
if basedir[len(basedir) - 1] is os.path.sep:
basedir = basedir[:-1]
return basedir
def __init__(self):
pool = Pool(processes=2)
self.graph = getGraph()
files = findFiles(opts)
self.progressQueue = Queue()
reporter = Process(target=ProgressReport,
args=(self.progressQueue, len(files)))
reporter.start()
result = pool.map(self.cacheFile, enumerate(files), chunksize=5)
self.progressQueue.put('END')
log.info("finished, %s results", len(result))
reporter.join()
def get_stats(): print 'Fetching NBA player stats...' stats_outfile = RUNDAY+'_nba_stats.csv' csvout = open(stats_outfile, 'wb') NUM_THREADS = 8 in_queue = Queue() out_queue = Queue() queue_players(in_queue) while not in_queue.empty(): jobs = [] for i in range(NUM_THREADS): if not in_queue.empty(): thread = Process(target=get_stats_helper, args=(in_queue, out_queue)) jobs.append(thread) thread.start() for thread in jobs: thread.join() while not out_queue.empty(): player = out_queue.get() del player['SUCCESS'] try: name = player['NAME'] except KeyError as e: continue player['TIME'] = RUNDAY fieldnames = [ 'TIME', 'NAME', 'JERSEY', 'SPORT', 'TEAM', 'POSITION', 'PTS', 'REB', 'AST', 'URL' ] csvwriter = csv.DictWriter(csvout, delimiter='|', fieldnames=fieldnames) csvwriter.writerow(player) csvout.close() print 'Finished fetching NBA player stats.' print 'Ouput saved in %s' % stats_outfile
def __init__(self, host, port, basedir):
self.host = host
self.port = port
self.basedir = self.__formatBaseDir(basedir)
self.socket = s.socket(s.AF_INET, s.SOCK_STREAM)
self.q = ThreadQue()
def __init__(self):
self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.port = settings.port
self.loginQueue = ThreadQue()
# encoding=utf-8
'''
@author: liangzai
pip install multiprocessing
'''
from multiprocessing.dummy import Pool, Queue
import threading
import time
# 队列
tasks = Queue()
for i in range(9):
tasks.put(str(i)*2)
def main(name):
while True:
time.sleep(1)
if name.empty():
print("name is over.")
break
else:
thread = threading.current_thread().getName()
print("线程：%s 打印：%s" % (thread, name.get()))
# 启动四个线程
pool = Pool(4, main, (tasks,))
while True:
time.sleep(5)
class seed(Process):
"""
Consumer thread that takes fuzzable requests from a Queue that's populated
by the crawl plugins and identified vulnerabilities by performing various
requests.
"""
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__(name='%sController' % self.get_name())
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def get_name(self):
return 'Seed'
def get_result(self, timeout=0.5):
return self._out_queue.get_nowait()
def has_pending_work(self):
return self._out_queue.qsize() != 0
def join(self):
return
@property
def out_queue(self):
# This output queue can contain one of the following:
# * POISON_PILL
# * (plugin_name, fuzzable_request, AsyncResult)
# * An ExceptionData instance
return self._out_queue
def terminate(self):
while True:
try:
self._out_queue.get_nowait()
except Empty:
break
else:
self._out_queue.task_done()
om.out.debug('No more tasks in Seed consumer output queue.')
def seed_output_queue(self, target_urls):
"""
Create the first fuzzable request objects based on the targets and put
them in the output Queue.
This will start the whole discovery process, since plugins are going
to consume from that Queue and then put their results in it again in
order to continue discovering.
"""
# We only want to scan pages that are in current scope
in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain()
for url in target_urls:
try:
#
# GET the initial target URLs in order to save them
# in a list and use them as our bootstrap URLs
#
response = self._w3af_core.uri_opener.GET(url, cache=True)
except ScanMustStopException, w3:
om.out.error('The target server is unreachable. Stopping.')
raise w3
except HTTPRequestException, hre:
msg = 'The target URL: "%s" is unreachable. Exception: "%s".'
om.out.error(msg % (url, hre))
except Exception, e:
msg = ('The target URL: "%s" is unreachable because of an'
' unhandled exception. Error description: "%s". See'
' debug output for more information.\n'
'Traceback for this error:\n%s')
om.out.error(msg % (url, e, traceback.format_exc()))
class seed(Process):
"""
Consumer thread that takes fuzzable requests from a Queue that's populated
by the crawl plugins and identified vulnerabilities by performing various
requests.
"""
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__(name='SeedController')
self.name = 'Seed'
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def get_result(self, timeout=0.5):
return self._out_queue.get_nowait()
def has_pending_work(self):
return self._out_queue.qsize() != 0
def join(self):
return
def terminate(self):
return
def seed_output_queue(self, target_urls):
"""
Create the first fuzzable request objects based on the targets and put
them in the output Queue.
This will start the whole discovery process, since plugins are going
to consume from that Queue and then put their results in it again in
order to continue discovering.
"""
# We only want to scan pages that are in current scope
in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain()
for url in target_urls:
# batman-fix already done verify, no need below codes
# try:
# #
# # GET the initial target URLs in order to save them
# # in a list and use them as our bootstrap URLs
# #
# response = self._w3af_core.uri_opener.GET(url, cache=True)
# except ScanMustStopException, w3:
# om.out.error('The target server is unreachable. Stopping.')
# raise w3
# except HTTPRequestException, hre:
# msg = 'The target URL: "%s" is unreachable. Exception: "%s".'
# om.out.error(msg % (url, hre))
# except Exception, e:
# msg = 'The target URL: "%s" is unreachable because of an' \
# ' unhandled exception. Error description: "%s". See' \
# ' debug output for more information.\n' \
# 'Traceback for this error:\n%s'
# om.out.error(msg % (url, e, traceback.format_exc()))
# else:
# _seed = FuzzableRequest(response.get_uri())
_seed = FuzzableRequest(url)
if in_scope(_seed):
self._out_queue.put((None, None, _seed))
# Update the set that lives in the KB
kb.kb.add_fuzzable_request(_seed)
self._out_queue.put(POISON_PILL)
class tizyoutubeproxy(object):
"""A class that accesses YouTube, retrieves stream URLs and creates and manages
a playback queue.
"""
def __init__(self):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
def set_play_mode(self, mode):
""" Set the playback mode.
:param mode: current valid values are "NORMAL" and "SHUFFLE"
"""
self.current_play_mode = getattr(self.play_modes, mode)
self.__update_play_queue_order()
def enqueue_audio_stream(self, arg):
"""Add the audio stream of a YouTube video to the
playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
yt_video = pafy.new(arg)
yt_audio = yt_video.getbestaudio(preftype="webm")
if not yt_audio:
raise ValueError(str("No WebM audio stream for : %s" % arg))
yt_info = VideoInfo(ytid=arg, title=yt_audio.title)
self.add_to_playback_queue(audio=yt_audio, video=yt_video, info=yt_info)
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Video not found : %s" % arg))
def enqueue_audio_playlist(self, arg):
"""Add all audio streams in a YouTube playlist to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info('arg : %s', arg)
try:
count = len(self.queue)
playlist = pafy.get_playlist2(arg)
if len(playlist) > 0:
for yt_video in playlist:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Playlist not found : %s" % arg))
def enqueue_audio_search(self, arg):
"""Search YouTube and add the audio streams to the
playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
query = generate_search_query(arg)
wdata = pafy.call_gdata('search', query)
wdata2 = wdata
count = 0
while True:
for track_info in get_tracks_from_json(wdata2):
self.add_to_playback_queue(info=track_info)
count += 1
if count > 100:
break
if not wdata2.get('nextPageToken'):
break
query['pageToken'] = wdata2['nextPageToken']
wdata2 = pafy.call_gdata('search', query)
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_mix(self, arg, feelinglucky=True):
"""Obtain a YouTube mix associated to a given video id or url and add all audio
streams in the mix playlist to the playback queue.
:param arg: a YouTube video id
:param feelinglucky: If True, it will perform another YouTube search to find
alternatives if the original mix cannot be found.
"""
logging.info('arg : %s', arg)
yt_video = None
try:
count = len(self.queue)
yt_video = pafy.new(arg)
playlist = yt_video.mix
if len(playlist) > 0:
for yt_video in playlist:
video_id = yt_video.videoid
video_title = yt_video.title
yt_info = VideoInfo(ytid=video_id, title=video_title)
self.add_to_playback_queue(video=yt_video, info=yt_info)
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except IndexError:
if not feelinglucky:
raise ValueError
else:
print_wrn("[YouTube] Could not find a mix for '{0}'. "\
"Searching YouTube instead. Feeling lucky?." \
.format(arg.encode('utf-8')))
if yt_video.title:
self.enqueue_audio_search(yt_video.title)
else:
self.enqueue_audio_stream(arg)
def enqueue_audio_mix_search(self, arg):
"""Obtain a YouTube mix associated to a given textual search and add all the
audio streams in the mix playlist to the playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
query = generate_search_query(arg)
wdata = pafy.call_gdata('search', query)
wdata2 = wdata
count = len(self.queue)
for track_info in get_tracks_from_json(wdata2):
if track_info and track_info.ytid:
try:
self.enqueue_audio_mix(track_info.ytid, feelinglucky=False)
break
except ValueError:
logging.info('Could not find a mix. Trying another video')
if count == len(self.queue):
raise ValueError
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_channel_uploads(self, arg):
"""Add all audio streams in a YouTube channel to the playback queue.
:param arg: a YouTube channel url
"""
logging.info('arg : %s', arg)
try:
count = len(self.queue)
channel = pafy.get_channel(arg)
if channel:
for yt_video in channel.uploads:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % arg))
def enqueue_audio_channel_playlist(self, channel_name, playlist_name):
"""Search a playlist within a channel and if found, adds all the audio streams
to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info('args : %s - %s', channel_name, playlist_name)
try:
count = len(self.queue)
channel = pafy.get_channel(channel_name)
if channel:
pl_dict = dict()
pl_titles = list()
pl_name = ''
playlist = None
for pl in channel.playlists:
print_nfo("[YouTube] [Playlist] '{0}'." \
.format(to_ascii(pl.title)))
if fuzz.partial_ratio(playlist_name, pl.title) > 50:
pl_dict[pl.title] = pl
pl_titles.append(pl.title)
if len(pl_titles) > 1:
pl_name = process.extractOne(playlist_name, pl_titles)[0]
playlist = pl_dict[pl_name]
elif len(pl_titles) == 1:
pl_name = pl_titles[0]
playlist = pl_dict[pl_name]
if pl_name:
if pl_name.lower() != playlist_name.lower():
print_wrn("[YouTube] Playlist '{0}' not found. " \
"Playing '{1}' instead." \
.format(to_ascii(playlist_name), \
to_ascii(pl_name)))
for yt_video in playlist:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % channel_name))
def current_audio_stream_title(self):
""" Retrieve the current stream's title.
"""
stream = self.now_playing_stream
title = ''
if stream:
title = to_ascii(stream['a'].title).encode("utf-8")
return title
def current_audio_stream_author(self):
""" Retrieve the current stream's author.
"""
stream = self.now_playing_stream
author = ''
if stream:
author = to_ascii(stream['v'].author).encode("utf-8")
return author
def current_audio_stream_file_size(self):
""" Retrieve the current stream's file size.
"""
stream = self.now_playing_stream
size = 0
if stream:
size = stream['a'].get_filesize()
return size
def current_audio_stream_duration(self):
""" Retrieve the current stream's duration.
"""
stream = self.now_playing_stream
duration = ''
if stream:
duration = to_ascii(stream['v'].duration).encode("utf-8")
return duration
def current_audio_stream_bitrate(self):
""" Retrieve the current stream's bitrate.
"""
stream = self.now_playing_stream
bitrate = ''
if stream:
bitrate = stream['a'].bitrate
return bitrate
def current_audio_stream_view_count(self):
""" Retrieve the current stream's view count.
"""
stream = self.now_playing_stream
viewcount = 0
if stream:
viewcount = stream['v'].viewcount
return viewcount
def current_audio_stream_description(self):
""" Retrieve the current stream's description.
"""
stream = self.now_playing_stream
description = ''
if stream:
description = to_ascii(stream['v'].description).encode("utf-8")
return description
def current_audio_stream_file_extension(self):
""" Retrieve the current stream's file extension.
"""
stream = self.now_playing_stream
file_extension = ''
if stream:
file_extension = to_ascii(stream['a'].extension).encode("utf-8")
return file_extension
def current_audio_stream_video_id(self):
""" Retrieve the current stream's video id.
"""
stream = self.now_playing_stream
video_id = ''
if stream:
video_id = to_ascii(stream['i'].ytid).encode("utf-8")
return video_id
def current_audio_stream_published(self):
""" Retrieve the current stream's upload date and time.
"""
stream = self.now_playing_stream
if stream:
published = to_ascii(stream['v'].published).encode("utf-8")
return published
def current_audio_stream_queue_index_and_queue_length(self):
""" Retrieve index in the queue (starting from 1) of the current stream and the
length of the playback queue.
"""
return self.queue_index + 1, len(self.queue)
def clear_queue(self):
""" Clears the playback queue.
"""
self.queue = list()
self.queue_index = -1
def remove_current_url(self):
"""Remove the currently active url from the playback queue.
"""
logging.info("")
if len(self.queue) and self.queue_index:
stream = self.queue[self.queue_index]
print_nfo("[YouTube] [Stream] '{0}' removed." \
.format(to_ascii(stream['i'].title).encode("utf-8")))
del self.queue[self.queue_index]
self.queue_index -= 1
if self.queue_index < 0:
self.queue_index = 0
self.__update_play_queue_order()
def next_url(self):
""" Retrieve the url of the next stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index += 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
next_stream = self.queue[self.play_queue_order \
[self.queue_index]]
return self.__retrieve_stream_url(next_stream, self.queue_index).rstrip()
else:
self.queue_index = -1
return self.next_url()
else:
return ''
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("KeyError, or AttributeError exception")
return self.next_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def prev_url(self):
""" Retrieve the url of the previous stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index -= 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
prev_stream = self.queue[self.play_queue_order \
[self.queue_index]]
return self.__retrieve_stream_url(prev_stream, self.queue_index).rstrip()
else:
self.queue_index = len(self.queue)
return self.prev_url()
else:
return ''
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("exception")
return self.prev_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def __update_play_queue_order(self):
""" Update the queue playback order.
A sequential order is applied if the current play mode is "NORMAL" or a
random order if current play mode is "SHUFFLE"
"""
total_streams = len(self.queue)
if total_streams:
if not len(self.play_queue_order):
# Create a sequential play order, if empty
self.play_queue_order = range(total_streams)
if self.current_play_mode == self.play_modes.SHUFFLE:
random.shuffle(self.play_queue_order)
print_nfo("[YouTube] [Streams in queue] '{0}'." \
.format(total_streams))
def __retrieve_stream_url(self, stream, queue_index):
""" Retrieve a stream url
"""
try:
if not len(self.workers):
for _ in range(WORKER_PROCESSES):
proc = Process(target=obtain_stream, \
args=(self.task_queue, \
self.done_queue)).start()
self.workers.append(proc)
while not self.done_queue.empty():
stream = self.done_queue.get()
self.queue[stream['q']] = stream
stream = self.queue[queue_index]
if not stream.get('v') or not stream.get('a'):
logging.info("ytid : %s", stream['i'].ytid)
video = stream.get('v')
if not video:
video = pafy.new(stream['i'].ytid)
audio = video.getbestaudio(preftype="webm")
if not audio:
logging.info("no suitable audio found")
raise AttributeError()
stream.update({'a': audio, 'v': video})
# streams = stream.get('v').audiostreams[::-1]
# pprint.pprint(streams)
# dump_stream_info(streams)
self.now_playing_stream = stream
return stream['a'].url.encode("utf-8")
except AttributeError:
logging.info("Could not retrieve the stream url!")
raise
def add_to_playback_queue(self, audio=None, video=None, info=None):
""" Add to the playback queue. """
if audio:
print_nfo("[YouTube] [Stream] '{0}' [{1}]." \
.format(to_ascii(audio.title).encode("utf-8"), \
to_ascii(audio.extension)))
if info:
print_nfo("[YouTube] [Stream] '{0}'." \
.format(to_ascii(info.title).encode("utf-8")))
queue_index = len(self.queue)
self.task_queue.put(dict(a=audio, v=video, i=info, q=queue_index))
self.queue.append(
dict(a=audio, v=video, i=info, q=queue_index))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__status_queue = Queue()
