def start(self):
self.workers = [
Worker(target=do_work, args=(
i,
self.queue,
)) for i in range(self.worker_num)
]
for w in self.workers:
w.start()
def crazy_callback(sleep_time):
logging.info('sleep time is %s' % sleep_time)
global probe
deep_probe = 1000
probe = 100
tn = Worker(name='thread_of_crazy_thread',
target=callback,
args=(sleep_time * 2, ))
tn.start()
time.sleep(sleep_time)
logging.info('and here I complete ... %s' % probe)
def _setup(self):
"""setup input/output queues and workers """
use_process = self._use_process
if use_process and sys.platform == "win32":
logger.info("Use multi-thread reader instead of "
"multi-process reader on Windows.")
use_process = False
bufsize = self._bufsize
if use_process:
from .shared_queue import SharedQueue as Queue
from multiprocessing import Process as Worker
from multiprocessing import Event
memsize = self._memsize
self._inq = Queue(bufsize, memsize=memsize)
self._outq = Queue(bufsize, memsize=memsize)
else:
if six.PY3:
from queue import Queue
else:
from Queue import Queue
from threading import Thread as Worker
from threading import Event
self._inq = Queue(bufsize)
self._outq = Queue(bufsize)
consumer_num = self._worker_num
id = str(uuid.uuid4())[-3:]
self._producer = threading.Thread(
target=self._produce,
args=('producer-' + id, self._source, self._inq))
self._producer.daemon = True
self._consumers = []
self._consumer_endsig = {}
for i in range(consumer_num):
consumer_id = 'consumer-' + id + '-' + str(i)
p = Worker(
target=self._consume,
args=(consumer_id, self._inq, self._outq, self._worker))
self._consumers.append(p)
p.daemon = True
setattr(p, 'id', consumer_id)
self._epoch = -1
self._feeding_ev = Event()
self._produced = 0 # produced sample in self._produce
self._consumed = 0 # consumed sample in self.next
def _setup(self):
"""setup input/output queues and workers """
use_process = False
if 'use_process' in self._worker_args:
use_process = self._worker_args['use_process']
bufsize = self._worker_args['bufsize']
if use_process:
from .shared_queue import SharedQueue as Queue
from multiprocessing import Process as Worker
from multiprocessing import Event
else:
if six.PY3:
from queue import Queue
else:
from Queue import Queue
from threading import Thread as Worker
from threading import Event
self._inq = Queue(bufsize)
self._outq = Queue(bufsize)
consumer_num = self._worker_args['worker_num']
id = str(uuid.uuid4())[-3:]
self._producer = threading.Thread(
target=self._produce,
args=('producer-' + id, self._source, self._inq))
self._producer.daemon = True
self._consumers = []
for i in range(consumer_num):
p = Worker(
target=self._consume,
args=('consumer-' + id + '_' + str(i), self._inq, self._outq,
self._mapper))
self._consumers.append(p)
p.daemon = True
self._epoch = -1
self._feeding_ev = Event()
self._produced = 0 # produced sample in self._produce
self._consumed = 0 # consumed sample in self.next
self._stopped_consumers = 0
def crazy_callback(sleep_time):
logging.info('sleep time is %s' % sleep_time)
global probe
deep_probe = 1000
probe = 100
tn = Worker(name='thread_of_crazy_thread',
target=callback,
args=(sleep_time * 2, ))
tn.start()
time.sleep(sleep_time)
logging.info('and here I complete ... %s' % probe)
w1 = Worker(name='test_thread1', target=callback, args=(1, ))
w2 = Worker(name='test_thread2', target=callback, args=(1, ))
w3 = Worker(name='test_thread3', target=callback, args=(1, ))
w4 = Worker(name='crazy_thread', target=crazy_callback, args=(1, ))
w1.start()
w2.start()
w3.start()
w4.start()
w1.join()
w2.join()
w3.join()
w4.join()
w4.terminate()
# for i in range(10):
# time.sleep(1)