class _Recognizer:
def __init__(self, index: int, callback: Callable[[Sound, int], None]):
self._logger = create_logger(f"Recognizer{index}")
self._barrier = Barrier(2)
self._index = index
self._callback = callback
self._stop = False
Thread(target=self._run).start()
def _run(self):
self._dejavu = Dejavu(dburl=f"sqlite:///bells{self._index}.sqlite")
while not self._stop:
self._barrier.wait()
self._recognize()
def recognize(self):
self._barrier.wait()
def _recognize(self):
song = self._dejavu.recognize(MicrophoneRecognizer,
seconds=_listen_seconds,
channels=1)
if song:
sound = Sound(song['song_id'])
confidence = song['confidence']
self._callback(sound, confidence)
def close(self):
self._stop = True
self._barrier.abort()
class SteppingMultiThreadedRunner(SteppingRunner):
"""
Has a clock thread, and a thread for each application process
in the system. The clock thread loops until stopped, waiting
for a barrier, after sleeping for remaining tick interval timer.
Application threads loop until stopped, waiting for the same
barrier. Then, after all threads are waiting at the barrier,
the barrier is lifted. The clock thread proceeds by sleeping
for the clock tick interval. The application threads proceed by
getting new notifications and processing all of them.
There are actually two barriers, so that each application thread
waits before getting notifications, and then waits for all processes
to complete getting notification before processing the notifications
through the application policy. This avoids events created by a process
application "bleeding" into the notifications of another process
application in the same clock cycle.
Todo:
Receive prompts, but set an event for the prompting process, to avoid unnecessary
runs.
Allow commands to be scheduled at future clock tick number, and execute when
reached.
"""
def __init__(self, *args: Any, **kwargs: Any):
super(SteppingMultiThreadedRunner, self).__init__(*args, **kwargs)
self.seen_prompt_events: Dict[str, Event] = {}
self.fetch_barrier: Optional[Barrier] = None
self.execute_barrier: Optional[Barrier] = None
self.application_threads: Dict[
str, BarrierControlledApplicationThread] = {}
self.clock_thread = None
self.stop_event = Event()
def handle_prompt(self, prompt: Prompt) -> None:
if isinstance(prompt, PromptToPull):
seen_prompt = self.seen_prompt_events[(prompt.process_name,
DEFAULT_PIPELINE_ID)]
seen_prompt.set()
def start(self) -> None:
super(SteppingMultiThreadedRunner, self).start()
parties = 1 + len(self.processes)
self.fetch_barrier = Barrier(parties)
self.execute_barrier = Barrier(parties)
# Create an event for each process.
for process_instance_id in self.processes:
self.seen_prompt_events[process_instance_id] = Event()
# Construct application threads.
for process_instance_id, process in self.processes.items():
thread = BarrierControlledApplicationThread(
process=process,
fetch_barrier=self.fetch_barrier,
execute_barrier=self.execute_barrier,
stop_event=self.stop_event,
)
self.application_threads[process_instance_id] = thread
# Start application threads.
for thread in self.application_threads.values():
thread.start()
# Start clock thread.
self.clock_thread = BarrierControllingClockThread(
normal_speed=self.normal_speed,
scale_factor=self.scale_factor,
tick_interval=self.tick_interval,
fetch_barrier=self.fetch_barrier,
execute_barrier=self.execute_barrier,
stop_event=self.stop_event,
is_verbose=self.is_verbose,
)
self.clock_thread.start()
def close(self) -> None:
self.stop_event.set()
super(SteppingMultiThreadedRunner, self).close()
if self.execute_barrier:
self.execute_barrier.abort()
if self.fetch_barrier:
self.fetch_barrier.abort()
for thread in self.application_threads.values():
thread.join(timeout=1)
if thread.is_alive():
print(f"Warning: application thread '"
f"{thread.app.name}"
f"' was still alive.")
self.application_threads.clear()
if self.clock_thread:
self.clock_thread.join(timeout=1)
if self.clock_thread.is_alive():
print(f"Warning: clock thread was still alive")
class ThreadPoolExecutor(object):
"""
Quenouille custom ThreadPoolExecutor able to lazily pull items to process
from iterated streams, all while bounding used memory and respecting
group parallelism and throttling.
Args:
max_workers (int, optional): Number of threads to use.
Defaults to min(32, os.cpu_count() + 1).
initializer (callable, optional): Function that will be run when starting
each worker thread. Can be useful to setup a threading local context
for instance.
initargs (iterable, optional): Arguments to pass to the thread initializer
function.
join (bool, optional): Whether to join worker threads on executor teardown.
Defaults to True.
daemonic (bool, optional): Whether to spawn daemon worker threads.
Defaults to False.
"""
def __init__(self,
max_workers=None,
initializer=None,
initargs=tuple(),
wait=True,
daemonic=False):
# Validation and defaults
validate_threadpool_kwargs('max_workers',
max_workers,
initializer=initializer,
initargs=initargs,
wait=wait,
daemonic=daemonic)
if max_workers is None:
max_workers = get_default_maxworkers()
# Properties
self.max_workers = max_workers
self.wait = wait
self.daemonic = daemonic
# Init
self.initializer = initializer
self.initargs = list(initargs)
# Queues
self.job_queue = Queue(maxsize=max_workers)
self.output_queue = Queue()
# Threading
self.throttled_groups = ThrottledGroups(self.output_queue)
self.teardown_event = Event()
self.teardown_lock = Lock()
self.broken_lock = Lock()
self.imap_lock = Lock()
self.boot_barrier = Barrier(max_workers + 1)
# State
self.closed = False
# Thread pool
self.threads = [
Thread(name='Thread-quenouille-%i-%i' % (id(self), n),
target=self.__worker,
daemon=self.daemonic) for n in range(max_workers)
]
# Actual initialization
try:
for thread in self.threads:
thread.start()
except BaseException:
self.boot_barrier.abort()
self.shutdown(wait=self.wait)
raise
try:
self.boot_barrier.wait()
except BrokenBarrierError:
self.shutdown(wait=self.wait)
raise BrokenThreadPool
def __enter__(self):
if self.closed:
raise RuntimeError('cannot re-enter a closed executor')
return self
def __exit__(self, *args):
self.shutdown(wait=self.wait)
def shutdown(self, wait=True):
if self.closed:
return
with self.teardown_lock:
self.teardown_event.set()
# Killing workers (cancel jobs and flushing end messages)
self.__kill_workers()
# Clearing the ouput queue since we won't be iterating anymore
self.__clear_output_queue()
# Clearing throttling state
self.throttled_groups.teardown()
# Waiting for worker threads to end
if wait:
for thread in self.threads:
if thread.is_alive():
thread.join()
self.closed = True
def __clear_output_queue(self):
clear(self.output_queue)
def __cancel_all_jobs(self):
clear(self.job_queue)
def __kill_workers(self):
self.__cancel_all_jobs()
n = sum(1 if t.is_alive() else 0 for t in self.threads)
flush(self.job_queue, n, THE_END_IS_NIGH)
def __worker(self):
# Thread initialization
try:
if self.initializer is not None:
self.initializer(*self.initargs)
self.boot_barrier.wait()
# NOTE: this naturally includes `BrokenBarrierError`
except BaseException:
self.boot_barrier.abort()
return
# Thread job consumer
try:
while not self.teardown_event.is_set():
try:
job = self.job_queue.get()
# Signaling we must tear down the worker thread
if job is THE_END_IS_NIGH:
break
# Actually performing the given task
job()
finally:
self.job_queue.task_done()
assert self.output_queue is not None
self.output_queue.put(job)
except BaseException as e:
smash(self.output_queue, e)
def __imap(self,
iterable,
func,
*,
ordered=False,
key=None,
parallelism=1,
buffer_size=DEFAULT_BUFFER_SIZE,
throttle=0):
# Cannot run in multiple threads
if self.imap_lock.locked():
raise RuntimeError(
'cannot run multiple executor methods concurrently from different threads'
)
# Cannot run if already closed
if self.closed:
raise RuntimeError('cannot use thread pool after teardown')
assert not self.boot_barrier.broken
self.imap_lock.acquire()
iterator = None
iterable_is_queue = is_queue(iterable)
if not iterable_is_queue:
iterator = iter(iterable)
# State
self.throttled_groups.update(key, throttle)
job_counter = count()
end_event = Event()
state = IterationState()
buffer = Buffer(self.throttled_groups,
maxsize=buffer_size,
parallelism=parallelism)
ordered_output_buffer = OrderedOutputBuffer()
def enqueue():
try:
while not end_event.is_set():
# First we check to see if there is a suitable buffered job
job = buffer.get()
if job is None:
# Else we consume the iterator to find one
try:
if not iterable_is_queue:
item = next(iterator)
else:
try:
item = iterable.get(block=False)
except Empty:
if state.has_running_tasks():
state.wait_for_any_task_to_finish()
continue
else:
raise StopIteration
except StopIteration:
if not buffer.empty():
continue
should_stop = state.declare_end()
# Sometimes the end of the iterator lags behind
# the output generator
if should_stop:
self.output_queue.put_nowait(THE_END_IS_NIGH)
break
group = None
if key is not None:
group = key(item)
job = Job(func,
item=item,
index=next(job_counter),
group=group)
buffer.put(job)
continue
# Registering the job
buffer.register_job(job)
state.start_task()
self.job_queue.put(job)
except BaseException as e:
smash(self.output_queue, e)
def cleanup(normal_exit=True):
end_event.set()
# Clearing the job queue to cancel next jobs
self.__cancel_all_jobs()
# Clearing the ouput queue since we won't be iterating anymore
self.__clear_output_queue()
# Sanity tests
if normal_exit:
assert buffer.is_clean()
assert ordered_output_buffer.is_clean()
# Detaching timer callback
self.throttled_groups.detach()
# Making sure we are getting rid of the dispatcher thread
if self.wait:
if dispatcher.is_alive():
dispatcher.join()
self.imap_lock.release()
def output():
raised = False
try:
while not state.should_stop() and not end_event.is_set():
job = self.output_queue.get()
if job is THE_END_IS_NIGH:
break
# Catching keyboard interrupts and other unrecoverable errors
if isinstance(job, BaseException):
raise job
# Unregistering job in buffer to let other threads continue working
buffer.unregister_job(job)
# Raising an error that occurred within worker function
# NOTE: shenanigans with tracebacks don't seem to change anything
if job.exc_info is not None:
raise job.exc_info[1].with_traceback(job.exc_info[2])
# Actually yielding the value to main thread
if ordered:
yield from ordered_output_buffer.output(job)
else:
yield job.result
# Acknowledging the job was finished
# NOTE: this was moved after yielding items so that the
# generator body may enqueue new jobs. It is possible that
# this has a slight perf hit if the body performs heavy work?
state.finish_task()
# Cleanup memory and avoid keeping references attached to
# ease up garbage collection
del job
except:
raised = True
raise
finally:
cleanup(not raised)
dispatcher = Thread(name='Thread-quenouille-%i-dispatcher' % id(self),
target=enqueue,
daemon=self.daemonic)
dispatcher.start()
return output()
def imap_unordered(self,
iterable,
func,
*,
key=None,
parallelism=1,
buffer_size=DEFAULT_BUFFER_SIZE,
throttle=0):
validate_imap_kwargs(iterable=iterable,
func=func,
max_workers=self.max_workers,
key=key,
parallelism=parallelism,
buffer_size=buffer_size,
throttle=throttle)
return self.__imap(iterable,
func,
ordered=False,
key=key,
parallelism=parallelism,
buffer_size=buffer_size,
throttle=throttle)
def imap(self,
iterable,
func,
*,
key=None,
parallelism=1,
buffer_size=DEFAULT_BUFFER_SIZE,
throttle=0):
validate_imap_kwargs(iterable=iterable,
func=func,
max_workers=self.max_workers,
key=key,
parallelism=parallelism,
buffer_size=buffer_size,
throttle=throttle)
return self.__imap(iterable,
func,
ordered=True,
key=key,
parallelism=parallelism,
buffer_size=buffer_size,
throttle=throttle)
class CPU:
def __init__(self, hilillos_to_run, quantum):
self.__pcb = PCBDataStructure()
self.threads_barrier = Barrier(2)
self.__dead_barrier = False
self.__killing_lock = Lock() # Lock used to kill the barrier
self.__waiting_lock = Lock()
self.__system_main_memory = MainMemory(self.__pcb, hilillos_to_run)
self.__simulation_statistics = SimulationStatistics()
self.__core0 = Core(0, self)
self.__core1 = Core(1, self)
self.__core_count = 2
self.running_cores = 2
self.__system_clock = 0
self.__default_quantum = quantum
self.__core_finished = False
self.__core_finished_counter = 0
# Data buss, instruction buss, cache 0, cache 1
self.__locks = [Lock(), Lock(), Lock(), Lock()]
self.__lock_owner = [-1, -1, -1, -1]
# Starts the cores for the simulation and prints statistics after the cores are finished
def start_cores(self):
self.__core1.start()
if self.__core_count > 1:
self.__core0.start()
thread = Thread(target=self.print_statistics(), args=())
thread.start()
# Print the statistics
def print_statistics(self):
self.__core0.join()
self.__core1.join()
self.__simulation_statistics.add_cache(0, self.__core0.get_data_cache())
self.__simulation_statistics.add_cache(1, self.__core1.get_data_cache())
self.__simulation_statistics.add_data_memory(self.__system_main_memory.get_data_memory())
self.__simulation_statistics.print_statistics()
print("Simulation Finished")
# Method to use the barrier
def wait(self):
if self.__core_count > 1:
if not self.__core_finished:
try:
barrier_thread_id = self.threads_barrier.wait()
if barrier_thread_id == 0:
self.__system_clock += 1
print("Ciclo de reloj: " + str(self.__system_clock))
except:
self.__system_clock += 1
print("Ciclo de reloj: " + str(self.__system_clock))
else:
self.__system_clock += 1
print("Ciclo de reloj: " + str(self.__system_clock))
if self.__core_finished_counter == 2:
self.__simulation_statistics.add_data_memory(self.__system_main_memory.get_data_memory())
# Method to kill the barrier
def kill_barrier(self):
self.__killing_lock.acquire(True)
if not self.__dead_barrier:
self.threads_barrier.abort()
self.__dead_barrier = True
self.__killing_lock.release()
# Method to acquire specific lock
def acquire__lock(self, lock_index, core_id):
if self.__locks[lock_index].acquire(False):
self.__lock_owner[lock_index] = core_id
return True
return False
# Method to release specific lock
def release_lock(self, lock_index):
self.__lock_owner[lock_index] = -1
self.__locks[lock_index].release()
# Method to release all the locks acquired by the core
def release_locks(self, core_id):
for index in range(0, 4):
if self.__lock_owner[index] == core_id:
self.__locks[index].release()
self.__lock_owner[index] = -1
# Method to get the PCB structure
def get_pcb_ds(self):
return self.__pcb
# Method to get the main memory
def get_main_memory(self):
return self.__system_main_memory
# Method to invalidate
# Receives the number of the core (0 or 1), and the memory_address of the block to change,
# and the new state of that block
def change_state_of_block_on_core_cache(self, core, memory_address, new_state):
if core == 0:
self.__core0.change_cache_block_state(memory_address, new_state)
else:
self.__core1.change_cache_block_state(memory_address, new_state)
# Return if the memory address its on the other core cache
def get_if_mem_address_is_on_core_cache(self, core, memory_address):
if core == 0 or self.__core_count <= 1:
return self.__core0.get_if_mem_address_is_on_self_cache(memory_address)
else:
return self.__core1.get_if_mem_address_is_on_self_cache(memory_address)
# Return the state of the memory address block on the core cache
def get_state_of_mem_address_on_core(self, core, memory_address):
if core == 0 or self.__core_count <= 1:
return self.__core0.get_memory_address_state_on_cache(memory_address)
else:
return self.__core1.get_memory_address_state_on_cache(memory_address)
# Method to store the cache block of the core on the main memory
def store_data_cache_block_on_mm_on_core(self, memory_address, cache_block_new_state, core):
if core == 0 or self.__core_count <= 1:
return self.__core0.store_data_cache_block_on_main_mem(memory_address, cache_block_new_state)
else:
return self.__core1.store_data_cache_block_on_main_mem(memory_address, cache_block_new_state)
# Method to invalidate RL on core, assumes that core has both cores and data bus locks
def invalidate_rl_on_core(self, mem_address, core):
if core == 0 or self.__core_count <= 1:
return self.__core0.invalidate_self_rl(mem_address)
else:
return self.__core1.invalidate_self_rl(mem_address)
# Method to set core finished bool to true
def notify_core_finished(self):
self.__core_finished = True
# Method to get the default quantum
def get_default_quantum(self):
return self.__default_quantum
# Method to get the simulation statistics
def get_simulation_statistics(self):
return self.__simulation_statistics
# Method to increase the finished cores counter
def increase_finished_counter(self):
self.__core_finished_counter += 1
class BarrierDemo(Thread):
def __init__(self, name, b):
Thread.__init__(self)
self.name = name
self.b = b
def run(self):
print("Thread name : ", self.name)
sleep(1)
print("Parties (number of threads) : ", b.parties)
sleep(2)
print(
"n_waiting (The number of threads currently waiting in the barrier) : ",
b.n_waiting)
b.wait()
if __name__ == "__main__":
b = Barrier(3)
t1 = BarrierDemo("Thread-1", b)
t2 = BarrierDemo("Thread-2", b)
t1.start()
sleep(1)
t2.start()
b.wait()
print("Barrier Broken (True if barrier is broken): ", b.broken)
sleep(1)
b.reset()
print("n_waiting after barrier.reset() call : ", b.n_waiting)
b.abort()
print("Barrier Aborted")
class TimedEventHandler(metaclass=Singleton):
def __init__(self):
self.__currentSimTime = None
self.__previousSimTime = None
self.__events = None
self.__subscribers = {}
self.__syncBarrier = Barrier(1) # (1): Simulator Control Blocks too
self.__syncLock = Lock()
self.__isStarted = False
self.__cleared = True
def cleanup(self):
self.__syncLock.acquire()
self.__syncLock.release()
def clear(self):
self.__init__()
def getCurrentSimTime(self):
self.__syncLock.acquire()
simtime = self.__currentSimTime
self.__syncLock.release()
return simtime
def getCurrentSimTimeStamp(self):
self.__syncLock.acquire()
simtime = self.__currentSimTime
self.__syncLock.release()
timestamp = TimeStamp(int(simtime), (simtime - int(simtime)) * 1000000)
return timestamp
def getPreviousSimTimeStamp(self):
self.__syncLock.acquire()
simtime = self.__previousSimTime
self.__syncLock.release()
timestamp = TimeStamp(int(simtime), (simtime - int(simtime)) * 1000000)
return timestamp
def getSimTimeDiff(self):
self.__syncLock.acquire()
if self.__previousSimTime is None:
timeDiff = None
else:
timeDiff = self.__currentSimTime - self.__previousSimTime
self.__syncLock.release()
return timeDiff
def updateSimStep(self, newSimTime):
self.__syncLock.acquire()
self.__previousSimTime = self.__currentSimTime
self.__currentSimTime = newSimTime.platform_timestamp
self.__notify()
self.__syncLock.release()
def start(self):
self.__syncLock.acquire()
self.__isStarted = True
self.__syncLock.release()
def stop(self):
self.__syncLock.acquire()
self.__syncBarrier.abort()
self.__notify()
self.__syncLock.release()
def subscribe(self, name, updateMethod):
self.__syncLock.acquire()
if self.__syncBarrier.n_waiting != 0:
raise Exception(
name, "tried to subscribe during runtime (syncBarrier has",
self.__syncBarrier.n_waiting, "threads waiting)")
if name in self.__subscribers:
raise Exception(name, "already subscribed")
else:
self.__syncBarrier = Barrier(self.__syncBarrier.parties + 1)
self.__subscribers[name] = updateMethod
self.__syncLock.release()
def syncBarrier(self):
self.__syncLock.acquire()
started = self.__isStarted
self.__syncLock.release()
if self.__isStarted:
self.__syncBarrier.wait(timeout=1.0)
else:
pass
def unsubscribe(self, name):
self.__syncLock.acquire()
if self.__syncBarrier.n_waiting != 0 and not self.__syncBarrier.broken:
raise Exception(
name, "tried to unsubscribe during runtime (syncBarrier has",
self.__syncBarrier.n_waiting, "threads waiting)")
del self.__subscribers[name]
self.__syncBarrier = Barrier(self.__syncBarrier.parties - 1)
self.__syncLock.release()
def __notify(self):
for subscriber, method in self.__subscribers.items():
# check events for subscriber
event = None
method(event)
from threading import Condition,Thread,Lock,Event,Barrier
import threading
import logging
FORMAT= '%(asctime)s %(threadName)s %(thread)d %(message)s'
logging.basicConfig(format=FORMAT,level=logging.INFO)
bar = Barrier(3) #number of parties,every 3 threads will start work
def worker(bar:Barrier):
logging.info("I am working-Number of waiting: {}".format(bar.n_waiting)) # numbers of threads stuck at barrier: once it;s 3, it will start
try:
bar.wait() # if timeout, barrier will abort and broken
except threading.BrokenBarrierError:
logging.info('Broken Error')
logging.info("Job done - Number of waiting: {}".format(bar.n_waiting))
for i in range(10): #10 threads
if i ==2 :
bar.abort() #abort: barrier is broken.
if i == 4:
bar.reset() #reset barrier
Thread(target=worker,args=(bar,),name='Barrier').start()
print('=====end========')
class farm:
'''init returns an object, we'll call o.run(i), then del o in the end.'''
def __init__(self,
num_threads,
init,
it,
extendable=False,
tn_tmpl=None,
reuse=None,
handler=None):
'''When extendable is True, it means that we need to leave threads ready in case the input list is extended (with extend()). Also the run() function can be invoked several times. The drawback is that underneath the input it is converted to a list, and then manipulated, so it's feasible to start with relatively small input data. Don't forget to call close(), it will clean up all the threads. Or you can use it as a context manager, so this will be called automatically upon __exit__().
If extendable is False, the code is simpler, but it supports iterables however big.
tn_tmpl is format() template with {} to be changed to thread's number.
reuse tells whether to stash worked sessions for future reuse. If it's a list, it's a global list of warm sessions.
A function in the handler parameter is invoked each second, while the processing is postponed, so it shouldn't take long to complete'''
self.waiting = 0
self.extendable = extendable
if self.extendable:
self.arr = list(
it) # If it was a generator for example. Make it real.
lg.info("Total: {}".format(len(self.arr)))
else: # Otherwise leave it as it is, we'll treat it as iterator
self.arr = iter(it)
self.cond = Condition()
self.num_threads = num_threads if num_threads else len(self.arr)
# Barrier to wait on for the restart (when extendable)
self.barr = Barrier(self.num_threads + 1)
self.init = init
self.tn_tmpl = tn_tmpl
self.reuse = reuse
self.handler = handler
# Objects living within threads, used to signal them to quit
# (by setting quit_flag)
self.objects = []
if type(reuse) is list:
self.reuse_pool = reuse
else:
self.reuse_pool = None
#if self.reuse: self.reuse_pool=[]
self.q = Queue()
self.tlist = []
def __del__(self):
# Let the user delete them if it was user-supplied
if type(self.reuse) is not list:
if self.reuse:
for i in self.reuse_pool:
del i
def close(self):
# Join our threads
if self.extendable:
lg.debug('Cancelling threads (break the barrier)')
self.barr.abort() # Let those waiting on the barrier to quit
lg.info('Joining threads')
for i in self.tlist:
i.join()
lg.info('Finished joining threads')
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
def extend(self, arr, end=False):
'''If end is True, add to the end'''
if not self.extendable:
lg.error(
'The farm is not extendable, "extendable" parameter is False')
return
with self.cond:
orig_len = len(self.arr)
#if type(arr) is GeneratorType: arr=tuple(arr)
if end or self.arr and self.arr[-1] is not None:
self.arr += arr
else: # When we're quitting
self.arr[:
0] = arr # Insert in the beginning, so poison pills won't get lost
lg.info("Total after extend: {}".format(len(self.arr)))
#lg.info("Notifying: {}".format(len(self.arr)-orig_len))
self.cond.notify(len(self.arr) - orig_len)
def print_arr(self):
'''Will be printed on farm exit. Beware to call it while the threads are running if you need the actual list.'''
with self.cond:
print(self.arr)
def reusing(self):
return type(self.reuse_pool) is list
def handle_item(o, i):
'''Handles the item with do() function of the class, passing parameters depending on the nature of the argument: can be presented as several arguments to make things easy. do() function must yield one or several results, that will be returned by farm.run()'''
lg.info('Item: {}'.format(i))
if isinstance(i, (tuple, list, set)): # Present as arguments
yield from o.do(*i)
else:
yield from o.do(i)
#lg.debug('do: {}'.format(res))
#return res
def do_extendable(self):
'''We need to leave threads ready in case the array is extended. Otherwise we can quit right after do() has completed. Also we can use this farm several times, the objects remain live, so we can extend and invoke another run() to gather the results as many times as we want.'''
o = self.init()
with suppress(AttributeError):
if not o.f: o.f = self # Set to the current farm
with self.cond:
self.objects.append(o)
while True:
self.cond.acquire()
self.waiting += 1
lg.debug('waiting incremented: {}, len={}'.format(
self.waiting, len(self.arr)))
if not len(self.arr):
if self.waiting == self.num_threads:
# No threads left to replenish the array, we should all quit
# Adding poison pills
if 1:
lg.info('Killing all')
# Put poison pills for everyone including us
self.arr += [None] * (self.num_threads)
self.cond.notify(
self.num_threads) # Wake up other threads
else:
lg.info('Killing all')
self.arr += [None] * (self.num_threads - 1)
self.cond.notify(self.num_threads - 1)
self.cond.release()
break
else:
self.cond.wait() # Someone else will kill us
lm = len(self.arr)
if lm: # Another check for those who have left cond.wait()
i = self.arr.pop()
self.waiting -= 1
lg.debug('waiting decremented: ' + str(self.waiting))
self.cond.release()
if not lm: continue # Someone has stolen our item, snap!
if i is None:
self.q.put(None) # Mark we're done
# Sleep on the condition to let other threads get their pills
lg.debug('Sleeping on barrier')
try:
self.barr.wait()
except BrokenBarrierError: # We're to quit
break
lg.debug('Continuing after barrier')
continue # then restart processing the queue
for j in farm.handle_item(o, i):
self.q.put(j)
with self.cond:
self.objects.remove(o)
del o
lg.info("has finished")
def do(self):
'''if an item from the iterator is a tuple, we explode it to be arguments to do(). Otherwise we pass it verbatim'''
#tracker = SummaryTracker()
o = None
if self.reusing(): # Try to get warm session
with self.cond:
if len(self.reuse_pool): o = self.reuse_pool.pop()
if not o: o = self.init()
with suppress(AttributeError):
if not o.f: o.f = self # Set to the current farm
with self.cond:
self.objects.append(o)
#lg.warning(len(self.arr))
while True:
with self.cond:
try:
i = next(self.arr)
except StopIteration: # empty
break
if i is None: break # End of queue marker
for j in farm.handle_item(o, i):
self.q.put(j)
#lg.error(asizeof.asizeof(o))
with self.cond:
self.objects.remove(o)
if self.reusing():
self.reuse_pool.append(o)
else:
del o # This will release proxy back to the pool
self.q.put(None) # Mark the thread has finished
lg.info("has finished")
#tracker.print_diff()
def cancel(self, cnt=0):
'''Cancels all the threads in the farm'''
# Put poison pills and then signal the threads to stop
if not cnt: cnt = self.num_threads # That many threads are running
with self.cond:
if self.extendable:
self.arr += [None] * cnt
else:
self.arr = chain(repeat(None, cnt), self.arr)
self.cond.notify(cnt)
for _ in self.objects:
_.quit_flag = True
def run(self):
'''Main function to invoke. When KeyboardInterrupt is received, it sets the quit_flag in all the objects present, retrievers then raise an exception. It's the problem of the do() function to handle it and possibly extend the main list with the item that wasn't handled to show it in the end (for possible restart)
self.handler() function is invoked each second if there are no items in the queue to allow for some rudimental auxiliary activity'''
# Now start the threads, only once
if self.tlist:
lg.debug('Restarting threads')
self.barr.wait()
else:
for i in range(self.num_threads):
tn = self.tn_tmpl.format(i) if self.tn_tmpl else None
t = Thread(
target=self.do_extendable if self.extendable else self.do,
name=tn)
self.tlist.append(t)
t.start()
cnt = self.num_threads # That many threads are running
while True:
try:
res = self.q.get(timeout=1)
except Empty:
if self.handler: self.handler()
continue # Go back to suck the queue
except KeyboardInterrupt:
lg.warning(
'Trying to kill nicely, putting {} None'.format(cnt))
self.cancel(cnt)
res = None
#lg.warning('run: {}'.format(res))
if res != None:
yield res
continue
cnt -= 1 # One thread has completed
if not cnt: # All threads are completed (but may still be processing the quit_flag), we'll join them in close()
return
