class ThreeFM(Process):
def __init__(self, d, name, year):
super().__init__()
self.d = d
self.name = name
self.year = year
self.r = []
self._mutex = RLock()
self._empty = Condition(self._mutex)
self._full = Condition(self._mutex)
def run(self):
with ThreadPoolExecutor(max_workers=2) as pool:
q = {
pool.submit(self.put, self.name, dat(self.year, m, 1)): m
for m in range(1, 13)
}
def __str__(self):
return str(self.d.values()[0])
def put(self, name, date):
with self._full:
while len(self.r) >= 12:
self._full.wait()
self.r.append(stat(date, name))
self.d[0] = self.r
self._empty.notify()
def get(self):
return self.d.values()[0]
class Producer(Process):
def __init__(self, prod_end, fname, SHARED_QUEUE_SIZE_LIMIT):
super(Producer, self).__init__()
self.prod_end = prod_end
self.fp = open(fname, 'r')
self.SHARED_QUEUE_SIZE_LIMIT = SHARED_QUEUE_SIZE_LIMIT
self.batch_queue = []
self.condition = Condition()
self.pipe_out_thread = PipeOutThread(prod_end, self.condition,
self.SHARED_QUEUE_SIZE_LIMIT,
self.batch_queue)
def _preprocess(self, data):
N = 1000 * 1000 * 10 * 3
while N > 0:
N -= 1
return data
def _is_shared_queue_full(self):
if self.pipe_out_thread.get_queue_size(
) >= self.SHARED_QUEUE_SIZE_LIMIT:
return True
else:
return False
def _preprocess_and_put_in_queue(self, data):
self.condition.acquire()
#print 'prod acquired'
if self._is_shared_queue_full():
#print 'prod: queue is full so waiting'
self.condition.wait()
self.batch_queue.append(self._preprocess(data))
#print 'self.batch_queue', len(self.batch_queue)
self.condition.notify()
self.condition.release()
def _read_data(self, i, dummy=False):
if dummy:
return 'soumya'
else:
data = None
offset = random.randint(5, 16) * GB
#print 'offset is : ' , offset/GB
self.fp.seek(offset)
data = self.fp.read(CHUNK_SIZE_TO_READ)
#print 'len_data ', len(data)
return data
def run(self):
self.pipe_out_thread.start()
for i in range(BATCHES):
data = self._read_data(i)
self._preprocess_and_put_in_queue(data)
#print 'prod put %s'%(i)
self.pipe_out_thread.join()
def setter(array: Array, cv: Condition):
"""
Sets the value of the first (and only) index
position in array exclusively and notifies the process
awaiting this action that a valid value is available.
:param array: a multiprocessing Array of integers of size 1 to which to write a value
:param cv: a Condition object (a condition variable) to allow the value to be read only when it is ready
"""
print('In setter.')
with cv:
print('Before setting:', array[0])
array[0] = 43
print('After setting:', array[0])
cv.notify()
class QueuedDistributor(BaseDistributor):
def __init__(self, socket_handler):
BaseDistributor.__init__(self, socket_handler)
self.socket_queue = SimpleQueue()
self.queue_condition = Condition(Lock())
def new_worker(self):
return Process(target=QueuedWorker,
args=(self.socket_handler, self.socket_queue,
self.queue_condition))
def process_handel(self, socket_handel):
self.queue_condition.acquire()
self.socket_queue.put(socket_handel)
self.queue_condition.notify()
self.queue_condition.release()
class ConditionVariableDemo:
"""
Demonstrates the use of condition variables within a class.
"""
def __init__(self):
self.array = Array('i', 1)
self.cv = Condition()
def run(self):
"""
Creates, runs, and joins a setter and a getter process.
"""
p0 = Process(target=self.setter)
p1 = Process(target=self.getter)
p0.start()
p1.start()
p0.join()
p1.join()
def setter(self):
"""
Sets the value of the first (and only) index
position in a exclusively and notifies the process
awaiting this action that a valid value is available.
:param a: a multiprocessing Array of integers of size 1 to which to write a value
:param cv: a Condition object (a condition variable) to allow the value to be read only when it is ready
"""
with self.cv:
print('In setter.')
print('Before setting:', self.array[0])
self.array[0] = 43
print('After setting:', self.array[0])
self.cv.notify()
def getter(self):
"""
Awaits notification through a Condition object
(condition variable) that a value is available in a.
Reads the value and prints it.
:param a: a multiprocessing Array of integers of size 1 from which to read a value
:param cv: a Condition object (a condition variable) to allow the value to be read only when it is ready
"""
print('In getter.')
with self.cv:
self.cv.wait_for(lambda: self.array[0] != 0)
print(f'Got {self.array[0]}.')
class Monitor:
def __init__(self, bufferSize):
# Shared Data
self.buffer = Array('i', bufferSize)
self.bufferSize = bufferSize
self.freePositions = Value('i', bufferSize)
# Local Data
self.nextRead = 0
self.nextWrite = 0
# Control Data
self.mutex = Lock()
self.items = Condition(self.mutex)
self.positions = Condition(self.mutex)
def produce(self, value):
self.mutex.acquire()
if (self.freePositions.value == 0):
self.positions.wait()
self.buffer[self.nextWrite] = value
self.nextWrite = (self.nextWrite + 1) % self.bufferSize
self.freePositions.value = self.freePositions.value - 1
self.items.notify()
self.mutex.release()
def consume(self):
newItem = None
self.mutex.acquire()
if (self.freePositions.value == self.bufferSize):
self.items.wait()
newItem = self.buffer[self.nextRead]
self.nextRead = (self.nextRead + 1) % self.bufferSize
self.freePositions.value = self.freePositions.value + 1
self.positions.notify()
self.mutex.release()
return newItem
class TProcessPoolServer(TServer):
def __init__(self, *args):
TServer.__init__(self, *args)
self.numWorkers = 10
self.workers = []
self.isRunning = Value('b', False)
self.stopCondition = Condition()
self.postForkCallback = None
def setPostForkCallback(self, callback):
if not callable(callback):
raise TypeError("This is not a callback!")
self.postForkCallback = callback
def setNumWorkers(self, num):
"""Set the number of worker threads that should be created"""
self.numWorkers = num
def workerProcess(self):
"""Loop getting clients from the shared queue and process them"""
if self.postForkCallback:
self.postForkCallback()
while self.isRunning.value:
try:
client = self.serverTransport.accept()
if not client:
continue
self.serveClient(client)
except (KeyboardInterrupt, SystemExit):
return 0
except Exception as x:
logger.exception(x)
def serveClient(self, client):
itrans = self.inputTransportFactory.getTransport(client)
otrans = self.outputTransportFactory.getTransport(client)
iprot = self.inputProtocolFactory.getProtocol(itrans)
oprot = self.outputProtocolFactory.getProtocol(otrans)
try:
while True:
self.processor.process(iprot, oprot)
except TTransportException:
pass
except Exception as x:
logger.exception(x)
itrans.close()
otrans.close()
def serve(self):
self.isRunning.value = True
self.serverTransport.listen()
for i in range(self.numWorkers):
try:
w = Process(target=self.workerProcess)
w.daemon = True
w.start()
self.workers.append(w)
except Exception as x:
logger.exception(x)
while True:
self.stopCondition.acquire()
try:
self.stopCondition.wait()
break
except (SystemExit, KeyboardInterrupt):
break
except Exception as x:
logger.exception(x)
self.isRunning.value = False
def stop(self):
self.isRunning.value = False
self.stopCondition.acquire()
self.stopCondition.notify()
self.stopCondition.release()
class RenderProcess:
"""
Wraps a multiprocessing.Process for rendering. Assumes there
is one MjSim per process.
"""
def __init__(self, device_id, setup_sim, update_sim, output_var_shape):
"""
Args:
- device_id (int): GPU device to use for rendering (0-indexed)
- setup_sim (callback): callback that is given a device_id and
returns a MjSim. It is responsible for making MjSim render
to given device.
- update_sim (callback): callback given a sim and device_id, and
should return a numpy array of shape `output_var_shape`.
- output_var_shape (tuple): shape of the synchronized output
array from `update_sim`.
"""
self.device_id = device_id
self.setup_sim = setup_sim
self.update_sim = update_sim
# Create a synchronized output variable (numpy array)
self._shared_output_var = Array(ctypes.c_double,
int(np.prod(output_var_shape)))
self._output_var = np.frombuffer(self._shared_output_var.get_obj())
# Number of variables used to communicate with process
self._cv = Condition()
self._ready = Value('b', 0)
self._start = Value('b', 0)
self._terminate = Value('b', 0)
# Start the actual process
self._process = Process(target=self._run)
self._process.start()
def wait(self):
""" Wait for process to be ready for another update call. """
with self._cv:
if self._start.value:
self._cv.wait()
if self._ready.value:
return
self._cv.wait()
def read(self, copy=False):
""" Reads the output variable. Returns a copy if copy=True. """
if copy:
with self._shared_output_var.get_lock():
return np.copy(self._output_var)
else:
return self._output_var
def update(self):
""" Calls update_sim asynchronously. """
with self._cv:
self._start.value = 1
self._cv.notify()
def stop(self):
""" Tells process to stop and waits for it to terminate. """
with self._cv:
self._terminate.value = 1
self._cv.notify()
self._process.join()
def _run(self):
sim = self.setup_sim(self.device_id)
while True:
with self._cv:
self._ready.value = 1
self._cv.notify_all()
with self._cv:
if not self._start.value and not self._terminate.value:
self._cv.wait()
if self._terminate.value:
break
assert self._start.value
self._start.value = 0
# Run the update and assign output variable
with self._shared_output_var.get_lock():
self._output_var[:] = self.update_sim(sim,
self.device_id).ravel()
class IODeviceManager(Thread):
def __init__(self, a_device, a_kernel, std_in=StandardInput(), std_out=StandardOutput()):
Thread.__init__(self)
self.set_device(a_device)
self.set_kernel(a_kernel)
self.set_input(std_in)
self.set_output(std_out)
self.set_mutex(RLock())
self.set_queue(SoQueue())
self.device_is_in_use = Condition(self.get_mutex())
self.the_queue_is_empty = Condition(self.get_mutex())
def get_kernel(self):
return self.kernel
def set_kernel(self, a_kernel):
self.kernel = a_kernel
def set_input(self, a_input):
self.std_in = a_input
def get_input(self):
return self.std_in
def set_output(self, a_output):
self.std_out = a_output
def get_output(self):
return self.std_out
def get_mutex(self):
return self.mutex
def set_mutex(self, a_mutex):
self.mutex = a_mutex
def get_queue(self):
return self.queue
def set_queue(self, a_queue):
self.queue = a_queue
def set_device(self, a_device):
self.device = a_device
self.get_device().set_device_manager(self)
def get_device(self):
return self.device
def the_device_is_busy(self):
with self.get_mutex():
return not self.get_device().is_not_busy()
def send_to_device(self):
with self.device_is_in_use:
while self.the_device_is_busy():
self.device_is_in_use.wait()
with self.get_mutex():
self.get_device().set_pcb(self.get())
self.get_device().process_pcb()
def notify_that_the_device_is_not_in_use(self):
with self.device_is_in_use:
self.device_is_in_use.notify()
def put(self, a_pcb):
with self.the_queue_is_empty:
with self.get_mutex():
self.get_queue().add_pcb(a_pcb)
self.the_queue_is_empty.notify()
def get(self):
with self.get_mutex():
return self.get_queue().get_first()
def queue_is_empty(self):
return self.get_queue().is_empty()
def send_io_end_interruption(self, a_pcb):
self.get_kernel().get_irq_manager().handle(Irq(IO_END_INTERRUPT, a_pcb))
def run(self):
while True:
with self.the_queue_is_empty:
while self.queue_is_empty():
self.the_queue_is_empty.wait()
self.send_to_device()
class TModelPoolServer(TServer):
''' A server runs a pool of multiple models to serve requests
Written by CongVm
'''
def __init__(self, handler, listModelConfig, *args, logger=None, timeout=0.1, batchSize=1):
TServer.__init__(self, *args)
self.timeout = timeout
self.batchSize = batchSize
if logger is not None:
self.logger = logger
else:
self.logger = logging.getLogger(__name__)
self.listModelConfig = listModelConfig
self.handler = handler
self.workers = []
self.isRunning = Value('b', False)
self.stopCondition = Condition()
self.postForkCallback = None
def setPostForkCallback(self, callback):
if not callable(callback):
raise TypeError("This is not a callback!")
self.postForkCallback = callback
def setListModelConfig(self, listModelConfig):
"""Set the number of worker threads that should be created"""
self.listModelConfig = listModelConfig
def workerProcess(self, kwargs):
"""Loop getting clients from the shared queue and process them"""
# Init Processor here
self.handlerInstance = self.handler(**kwargs)
self.procInstance = self.processor(self.handlerInstance)
if self.postForkCallback:
self.postForkCallback()
listClient = []
t = time()
while self.isRunning.value:
try:
client = self.serverTransport.accept()
if not client:
continue
listClient.append(client)
if len(listClient) >= self.batchSize or time() - t >= self.timeout:
self.serveClient(self.procInstance, listClient)
listClient.clear()
t = time()
except (KeyboardInterrupt, SystemExit):
return 0
except Exception as x:
self.logger.exception(x)
def parseClients(self, listClient):
listOtrans = []
listItrans = []
listIprot = []
listOprot = []
for client in listClient:
itrans = self.inputTransportFactory.getTransport(client)
otrans = self.outputTransportFactory.getTransport(client)
iprot = self.inputProtocolFactory.getProtocol(itrans)
oprot = self.outputProtocolFactory.getProtocol(otrans)
listOtrans.append(otrans)
listItrans.append(itrans)
listIprot.append(iprot)
listOprot.append(oprot)
return listOtrans, listItrans, listIprot, listOprot
def serveClient(self, procInstance, listClient):
"""Process input/output from a client for as long as possible"""
listOtrans, listItrans, listIprot, listOprot = self.parseClients(listClient)
try:
while True:
procInstance.process(listIprot, listOprot)
except TTransportException:
pass
except Exception as x:
self.logger.exception(x)
for itrans, otrans in zip(listItrans, listOtrans):
itrans.close()
otrans.close()
def serve(self):
"""Start workers and put into queue"""
# this is a shared state that can tell the workers to exit when False
self.isRunning.value = True
# first bind and listen to the port
self.serverTransport.listen()
# fork the children
for modelConfig in self.listModelConfig:
try:
w = Process(target=self.workerProcess, args=(modelConfig, ))
w.daemon = True
w.start()
self.workers.append(w)
except Exception as x:
self.logger.exception(x)
# wait until the condition is set by stop()
while True:
self.stopCondition.acquire()
try:
self.stopCondition.wait()
break
except (SystemExit, KeyboardInterrupt):
break
except Exception as x:
self.logger.exception(x)
self.isRunning.value = False
def stop(self):
self.isRunning.value = False
self.stopCondition.acquire()
self.stopCondition.notify()
self.stopCondition.release()
class TProcessPoolServer(TServer):
"""Server with a fixed size pool of worker subprocesses to service requests
Note that if you need shared state between the handlers - it's up to you!
Written by Dvir Volk, doat.com
"""
def __init__(self, *args):
TServer.__init__(self, *args)
self.numWorkers = 10
self.workers = []
self.isRunning = Value('b', False)
self.stopCondition = Condition()
self.postForkCallback = None
def setPostForkCallback(self, callback):
if not callable(callback):
raise TypeError("This is not a callback!")
self.postForkCallback = callback
def setNumWorkers(self, num):
"""Set the number of worker threads that should be created"""
self.numWorkers = num
def workerProcess(self):
"""Loop getting clients from the shared queue and process them"""
if self.postForkCallback:
self.postForkCallback()
while self.isRunning.value:
try:
client = self.serverTransport.accept()
if not client:
continue
self.serveClient(client)
except (KeyboardInterrupt, SystemExit):
return 0
except Exception as x:
logger.exception(x)
def serveClient(self, client):
"""Process input/output from a client for as long as possible"""
itrans = self.inputTransportFactory.getTransport(client)
otrans = self.outputTransportFactory.getTransport(client)
iprot = self.inputProtocolFactory.getProtocol(itrans)
oprot = self.outputProtocolFactory.getProtocol(otrans)
try:
while True:
self.processor.process(iprot, oprot)
except TTransportException as tx:
pass
except Exception as x:
logger.exception(x)
itrans.close()
otrans.close()
def serve(self):
"""Start workers and put into queue"""
# this is a shared state that can tell the workers to exit when False
self.isRunning.value = True
# first bind and listen to the port
self.serverTransport.listen()
# fork the children
for i in range(self.numWorkers):
try:
w = Process(target=self.workerProcess)
w.daemon = True
w.start()
self.workers.append(w)
except Exception as x:
logger.exception(x)
# wait until the condition is set by stop()
while True:
self.stopCondition.acquire()
try:
self.stopCondition.wait()
break
except (SystemExit, KeyboardInterrupt):
break
except Exception as x:
logger.exception(x)
self.isRunning.value = False
def stop(self):
self.isRunning.value = False
self.stopCondition.acquire()
self.stopCondition.notify()
self.stopCondition.release()
class MultiProcessScheduler:
def __init__(self):
self.cond = Condition() # default to RLock
# If duplex is False then the pipe is unidirectional
# conn1 for receiving messages and conn2 for sending messages.
conn1, conn2 = Pipe(duplex=False)
self.connREAD = conn1
self.connWRITE = conn2
# a holder to the closest task to execute
# it is not safe to access this variable directly as
# there might be data on the pipe, use self.__getClosestTask()
self._closestTask = None
# multiprocessing.Queue is used here to exchange task between the add
# call and the service running __run() method
self.queue = SimpleQueue()
# dummy Process, the correct one will be created when the first
# task is added
self.service = Process()
# TODO create destructor to avoid leaving with items on queue
def __getClosestTask(self):
'''
return the closest task to execute (i.e., top on pq)
'''
if self.connREAD.poll():
ret = None
while self.connREAD.poll():
ret = self.connREAD.recv()
self._closestTask = ret
print("[conn] closestTaskUpdate: ", self._closestTask)
return self._closestTask
def add(self, task):
if type(task) is not Task:
raise TypeError
self.queue.put(task)
if not self.service.is_alive():
# it seams that Process.run() is a blocking call
# so the only way to re-run the process is to create another one
self.service = Process(target=MultiProcessScheduler.__run,
args=(self.cond, self.queue,
self.connWRITE),
daemon=False)
self.service.start()
else:
# notify the condition variable if the new task has the
# closest execution time
closestTask = self.__getClosestTask()
if closestTask and task.time < closestTask.time:
self.cond.acquire()
self.cond.notify()
self.cond.release()
@staticmethod
def __run(cond, queue, conn):
tasksQueue = []
print("[run] starting", queue.empty())
while True:
# remove tasks from queue and add to
# internal priorityQueue (tasksQueue)
while not queue.empty():
task = queue.get()
heappush(tasksQueue, task)
print("[run] adding task to pq: ", task)
# if there are task on the priority queue,
# check when the closest one should be runned
if tasksQueue:
etime, _, _ = task = tasksQueue[0]
now = time()
if etime < now:
# only pop before running
# if a task is not being running in a given time,
# the next this loop runs that task might not be the
# closest one
_, fn, args = heappop(tasksQueue)
print("[run] running:", task)
p = Process(target=fn, args=args, daemon=False)
p.start()
else:
delay = etime - now
print("[run] sleeping for ", delay, task)
# send the closest task to the pipe
conn.send(task)
cond.acquire()
cond.wait(timeout=delay)
if not tasksQueue and queue.empty():
# only stop the service if there are no task anwhere
break
print("[run] done")
def solve(max_level, goal, num_workers):
# prepare message queue shared with workers
tasks = Queue()
task_lock = Lock()
task_cv = Condition(lock=task_lock)
# create and start workers
workers = []
for i in range(0, num_workers):
solutions = set()
parent_conn, child_connn = Pipe()
worker = Process(target=run_worker,
args=(child_connn, goal, max_level, tasks, task_lock,
task_cv))
worker.start()
workers.append((worker, parent_conn))
# Find all possible sequences: [n0, n1, n2, ..., nM] (M=max_level)
# where nX is the number of binary operators so that
# '1 <n0 ops> 2 <n1 ops> 3 <n2 ops> ... M+1 <nM ops>' can be a valid
# Reverse Polish Notation. Key conditions are:
# 1. n0 + n1 + ... + nM = M
# 2. for any X, n0 + n1 + ... + nX <= X
# (Note that from condition #2 n0 is always 0.)
# We'll build the sequences in 'numops_list' below while exploring cases
# in a BFS-like (or DP-like) manner.
# This is a queue to maintain outstanding search results. Its each element
# is a tuple of 2 items: 'numops_list', 'total_ops'
# A tuple of (N, T) means:
# - N = [n0, n1, ..., nX]
# - T = sum(N)
# (Note that we don't necessarily have to keep T as it can be derived
# from N. But we do this for efficiency).
# The search is completed when len(N) reaches M (i.e., X=M-1) by appending
# the last item of nM = M - (n0 + n1 + ... + nX) = M - T (see condition #1).
tmp = [([0], 0)]
while tmp:
numops_list, total_ops = tmp.pop(0)
level = len(numops_list)
if level < max_level:
# Expand the sequence with all possible numbers of operators at
# the current level so we can explore the next level for each of
# them.
for i in range(0, level - total_ops + 1): # see condition #2
tmp.append((numops_list + [i], total_ops + i))
else:
# Found one valid RPN template. Pass it to workers and have them
# work on it.
numops_list.append(max_level - total_ops)
with task_lock:
tasks.put(numops_list)
task_cv.notify()
# Tell workers all data have been passed.
solutions = set()
with task_lock:
for _ in workers:
tasks.put(None)
task_cv.notify_all()
# Wait until all workers complete the tasks, while receiving any
# intermediate and last solutions. The received solutions may not
# necessarily be fully unique, so we have to unify them here, again.
# Received data of 'None' means the corresponding worker has completed
# its task.
# Note: here we assume all workers are reasonably equally active in
# sending data, so we simply perform blocking receive.
conns = set([w[1] for w in workers])
while conns:
for c in conns.copy():
worker_data = c.recv()
if worker_data is None:
conns.remove(c)
continue
for solution in worker_data:
if solution not in solutions:
solutions.add(solution)
# All workers have completed. Cleanup them and print the final unified
# results. If we are to show all expressions (i.e. goal is None), sort
# results by the expressions' values (listing integers followed by all
# non-integers, followed by 'divided by 0' cases.
for w in workers:
w[0].join()
if goal is None:
l = list(solutions)
l.sort(
key=lambda x: (0, x[0]) if type(x[0]) == int else (1, str(x[0])))
for solution in l:
print('%s = %s' % (solution[1], str(solution[0])))
else:
for solution in solutions:
print(solution)
from config import TASK_QUEUE_SIZE, UPDATE_TIME
if __name__ == "__main__":
myip = getMyIP()
DB_PROXY_NUM = Value('i', 0)
q1 = Queue(maxsize=TASK_QUEUE_SIZE)
q2 = Queue()
sleep_condition = Condition(Lock())
p0 = Process(target=start_api_server, args=(sleep_condition, ))
p1 = Process(target=startProxyCrawl, args=(q1, DB_PROXY_NUM, myip, sleep_condition))
p2 = Process(target=validator, args=(q1, q2, myip))
p3 = Process(target=store_data, args=(q2, DB_PROXY_NUM))
p0.start()
p1.start()
p2.start()
p3.start()
while True:
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
print("sleep start:" + now)
time.sleep(UPDATE_TIME)
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
print("sleep finish:" + now)
sleep_condition.acquire()
sleep_condition.notify()
sleep_condition.release()
p0.join()
p1.join()
p2.join()
p3.join()
:param num: die Prozessnummer
:return: None
"""
with condition:
# Auf Signal warten
condition.wait()
with open("foo.txt", "ab") as fo:
for i in range(100):
fo.write("Process ".encode())
fo.flush()
fo.write(str(num).encode())
fo.flush()
fo.write(": Hello!\n".encode())
fo.flush()
condition.notify()
if __name__ == "__main__":
# File loeschen, falls es bereits existiert
if os.path.isfile("foo.txt"):
os.remove("foo.txt")
condition = Condition()
# Zehn Prozesse erstellen und starten
for num in range(10):
Process(target=sayhello, args=(condition, num)).start()
time.sleep(1)
# Den Vorgang "anstossen", da ja alle Prozesse
# zu Beginn warten
with condition:
condition.notify()
class DataLoader:
""" Class for loading data
Attributes:
num_processor: an integer indicating the number of processors
for loading the data, normally 4 is enough
capacity: an integer indicating the capacity of the data load
queue, default set to 10
batch_size: an integer indicating the batch size for each
extraction from the data load queue
phase: an string indicating the phase of the data loading process,
can only be 'train' or 'test'
"""
def __init__(self,
num_processor,
batch_size,
phase,
batch_idx_init=0,
data_ids_init=train_ids,
capacity=10):
self.num_processor = num_processor
self.batch_size = batch_size
self.data_load_capacity = capacity
self.manager = Manager()
self.batch_lock = Lock()
self.mutex = Lock()
self.cv_full = Condition(self.mutex)
self.cv_empty = Condition(self.mutex)
self.data_load_queue = self.manager.list()
self.cur_batch = self.manager.list([batch_idx_init])
self.processors = []
if phase == 'train':
self.data_ids = self.manager.list(data_ids_init)
elif phase == 'test':
self.data_ids = self.manager.list(test_ids)
else:
raise ValueError('Could not set phase to %s' % phase)
def __load__(self):
while True:
image_dicts = []
self.batch_lock.acquire()
image_ids = self.data_ids[self.cur_batch[0] *
self.batch_size:(self.cur_batch[0] + 1) *
self.batch_size]
self.cur_batch[0] += 1
if (self.cur_batch[0] + 1) * self.batch_size >= len(self.data_ids):
self.cur_batch[0] = 0
random.shuffle(self.data_ids)
self.batch_lock.release()
self.cv_full.acquire()
if len(self.data_load_queue) > self.data_load_capacity:
self.cv_full.wait()
self.data_load_queue.append(get_data(image_ids))
self.cv_empty.notify()
self.cv_full.release()
def start(self):
for _ in range(self.num_processor):
p = Process(target=self.__load__)
p.start()
self.processors.append(p)
def get_batch(self):
self.cv_empty.acquire()
if len(self.data_load_queue) == 0:
self.cv_empty.wait()
batch_data = self.data_load_queue.pop()
self.cv_full.notify()
self.cv_empty.release()
return batch_data
def get_status(self):
self.batch_lock.acquire()
current_cur_batch = self.cur_batch[0]
current_data_ids = self.data_ids
self.batch_lock.release()
return {
'batch_idx': int(current_cur_batch),
'data_ids': list(current_data_ids)
}
def stop(self):
for p in self.processors:
p.terminate()
def solve(max_level, goal, num_workers):
# prepare message queue shared with workers
tasks = Queue()
task_lock = Lock()
task_cv = Condition(lock=task_lock)
# create and start workers
workers = []
for i in range(0, num_workers):
solutions = set()
parent_conn, child_connn = Pipe()
worker = Process(target=run_worker,
args=(child_connn, goal, max_level, tasks,
task_lock, task_cv))
worker.start()
workers.append((worker, parent_conn))
# Find all possible sequences: [n0, n1, n2, ..., nM] (M=max_level)
# where nX is the number of binary operators so that
# '1 <n0 ops> 2 <n1 ops> 3 <n2 ops> ... M+1 <nM ops>' can be a valid
# Reverse Polish Notation. Key conditions are:
# 1. n0 + n1 + ... + nM = M
# 2. for any X, n0 + n1 + ... + nX <= X
# (Note that from condition #2 n0 is always 0.)
# We'll build the sequences in 'numops_list' below while exploring cases
# in a BFS-like (or DP-like) manner.
# This is a queue to maintain outstanding search results. Its each element
# is a tuple of 2 items: 'numops_list', 'total_ops'
# A tuple of (N, T) means:
# - N = [n0, n1, ..., nX]
# - T = sum(N)
# (Note that we don't necessarily have to keep T as it can be derived
# from N. But we do this for efficiency).
# The search is completed when len(N) reaches M (i.e., X=M-1) by appending
# the last item of nM = M - (n0 + n1 + ... + nX) = M - T (see condition #1).
tmp = [([0], 0)]
while tmp:
numops_list, total_ops = tmp.pop(0)
level = len(numops_list)
if level < max_level:
# Expand the sequence with all possible numbers of operators at
# the current level so we can explore the next level for each of
# them.
for i in range(0, level - total_ops + 1): # see condition #2
tmp.append((numops_list + [i], total_ops + i))
else:
# Found one valid RPN template. Pass it to workers and have them
# work on it.
numops_list.append(max_level - total_ops)
with task_lock:
tasks.put(numops_list)
task_cv.notify()
# Tell workers all data have been passed.
solutions = set()
with task_lock:
for _ in workers:
tasks.put(None)
task_cv.notify_all()
# Wait until all workers complete the tasks, while receiving any
# intermediate and last solutions. The received solutions may not
# necessarily be fully unique, so we have to unify them here, again.
# Received data of 'None' means the corresponding worker has completed
# its task.
# Note: here we assume all workers are reasonably equally active in
# sending data, so we simply perform blocking receive.
conns = set([w[1] for w in workers])
while conns:
for c in conns.copy():
worker_data = c.recv()
if worker_data is None:
conns.remove(c)
continue
for solution in worker_data:
if solution not in solutions:
solutions.add(solution)
# All workers have completed. Cleanup them and print the final unified
# results. If we are to show all expressions (i.e. goal is None), sort
# results by the expressions' values (listing integers followed by all
# non-integers, followed by 'divided by 0' cases.
for w in workers:
w[0].join()
if goal is None:
l = list(solutions)
l.sort(key=lambda x: (0, x[0]) if type(x[0]) == int else (1, str(x[0])))
for solution in l:
print('%s = %s' % (solution[1], str(solution[0])))
else:
for solution in solutions:
print(solution)
class Orchestrator:
def __init__(self,
submission_queue: multiprocessing.Queue,
status_provider: BatchStatusProvider,
config_file: str,
strict_config: bool,
log_folder: str,
cache_search_dirs: List[str],
log_event_que: LogEventQueue,
singleton_run_summary_path: Optional[str] = None):
self._submission_que: multiprocessing.Queue = submission_queue
self._status_provider: BatchStatusProvider = status_provider
self._config_file: str = config_file
self._strict_config: bool = strict_config
self._log_folder: str = log_folder
self._cache_search_dirs = cache_search_dirs
self._log_event_que = log_event_que
self._singleton_run_summary_path = singleton_run_summary_path
self._on_batch_id = -1
self._on_batch_type: type = type(None)
self._master_thread = Thread(target=self._master_thread_loop,
name="OrchestratorMasterThread",
args=(()),
daemon=True)
self._run_summary_thread_gate = Event()
self._run_summary_thread = Thread(target=self._run_summary_loop,
name="OrchestratorRunSummaryThread",
args=(()),
daemon=True)
self.__debug_loop_thread = Thread(target=self.__debug_loop,
name="OrchestratorDebugLoop",
args=(()),
daemon=True)
#TODO(andwald): The following hypothetical thread dynamically sets RTF and Concurrency of EndpointManagers
# according to its own decoupled logic. This will be nice and pluggable since EndpointManagers
# already adhere to whatever the dynamic settings are for the Atomic Shared Variables of
# RTF and Concurrency, which is what this thread will manipulate.
# self._perf_thread = Thread(target=self.perf_thread_loop, name="OrchestratorPerfThread", args=(self,), daemon=True)
self._file_queue = Queue()
self._file_queue_size = 0
self._in_progress: Dict[str, WorkItemRequest] = {
} # WorkItemRequest.filepath -> WorkItemRequest
self._in_progress_owner: Dict[str, EndpointManager] = {
} # WorkItemRequest.filepath -> EndpointManager
self._work_results: Dict[str, WorkItemResult] = {
} # WorkItemRequest.filepath -> WorkItemResult
self._batch_completion_evt = Event()
self._accounting_lock = RLock()
self._file_queue_cond = Condition(self._accounting_lock)
self._run_summary_lock = Lock()
self._misc_lock = Lock()
self._summarizer: BatchRunSummarizer = None
self._stop_requested = False
self._endpoint_managers: List[EndpointManager] = []
self._endpoint_generation = 0
self._old_managers = set(
) # Set[str], contains names of now-inactive endpoint managers
self._config_notifier: ThreadedNotifier = \
notify_file_modified(self._config_file, self.hotswap_endpoint_managers, self._log_event_que)
self._start_time = time.time()
self._creator_pid = current_process().pid
logger.info("Orchestrator created by process: {0}".format(
self._creator_pid))
self.__cnt_work_success_cb = 0
self.__cnt_work_failure_cb = 0
self._master_thread.start()
self._run_summary_thread.start()
# self.__debug_loop_thread.start() # Enable to debug concurrency changes.
def is_alive(self):
return self._master_thread.is_alive()
def join(self):
self._master_thread.join()
def _run_summary_loop(self):
while not self._stop_requested:
# Prevent redundant updates when nothing can change.
self._run_summary_thread_gate.wait()
if self._stop_requested:
return
if self._on_batch_id > -1 and self._summarizer:
try:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=False)
# Don't ever let this thread die as it's too important.
# Log and re-try. Repetitive failure loop will at least get logged.
except Exception as e:
exception_details = traceback.format_exc()
logger.error(
"Orchestrator: run_summary_thread in run_summary_loop(): "
"Caught {0}, \nDetails: {1}".format(
type(e).__name__, exception_details))
time.sleep(RUN_SUMMARY_LOOP_INTERVAL)
def __debug_loop(self):
"""
This is only intended to be used during development and debugging.
"""
def _check_lock_acq(lock):
acquired = lock.acquire(block=False)
if acquired:
lock.release()
return False
# We weren't able to acquire, so it's taken
return True
# Loop forever. This is a daemonic thread and it will intentionally
# only die when the process owning Orchestrator dies.
last_cnt_work_success = 0
while True:
logger.debug("Stop requested: {0}".format(self._stop_requested))
logger.debug("Batch que size: {0}".format(
self._submission_que.qsize()))
logger.debug("On batch id: {0}".format(self._on_batch_id))
logger.debug("File queue size: {0}".format(self._file_queue_size))
logger.debug("Num in progress: {0}".format(len(self._in_progress)))
logger.debug("Orchestrator accounting lock taken: {0}".format(
_check_lock_acq(self._accounting_lock)))
logger.debug("Status provider accounting lock taken: {0}".format(
_check_lock_acq(BatchStatusProvider.lock)))
logger.debug(
"Notify work success callback entry count: {0}".format(
self.__cnt_work_success_cb))
logger.debug(
"Work items completed since last debug print: {0}".format(
self.__cnt_work_success_cb - last_cnt_work_success))
last_cnt_work_success = self.__cnt_work_success_cb
logger.debug(
"Notify work failure callback entry count: {0}".format(
self.__cnt_work_failure_cb))
logger.debug("Run summary thread alive: {0}".format(
self._run_summary_thread.is_alive()))
logger.debug("Number of old endpoint managers: {0}".format(
len(self._old_managers)))
for epm in self._endpoint_managers:
logger.debug("Endpoint manager: {0}".format(epm.name))
logger.debug(" Current requests: {0}".format(
epm._current_requests))
logger.debug(" Current requests lock taken: {0}".format(
_check_lock_acq(epm._current_requests_lock)))
logger.debug(" Pool apply_async count: {0}".format(
epm._cnt_apply_async))
logger.debug(" Pool callback count: {0}".format(
epm._cnt_pool_cb))
logger.debug(" Pool callback returns count: {0}".format(
epm._cnt_pool_cb_rets))
logger.debug(" Stop requested: {0}".format(
epm._stop_requested))
logger.debug(" Now trying to steal work: {0}".format(
epm._in_steal_work_fn))
logger.debug("Stack frames of all threads:")
logger.debug("\n*** STACKTRACE - START ***\n")
current_threads_stacktrace(use_logger=True)
logger.debug("\n*** STACKTRACE - END ***\n")
time.sleep(DEBUG_LOOP_INTERVAL)
def write_summary_information(self,
write_run_summary: bool = True,
log_conclusion_msg: bool = False,
allow_fail: bool = False):
"""
Summarize individual file results, along with overall results,
and write them to log and/or file. Also log a conclusion message
if requested.
:param write_run_summary: whether run summary (individual files + overall)
should be written to file.
:param log_conclusion_msg: whether a conclusion message should be logged
which includes final stats and lists failures.
:param allow_fail: log failure to write run summary but do not raise exception.
"""
# To ensure history serialization, we wrap this method
# in its own lock that nobody else contends with except for
# the threads that invoke this.
with self._run_summary_lock:
# Take a consistent snapshot and then report on the snapshot
# without holding back forward progress.
with self._accounting_lock:
snap_work_results: Dict[str, WorkItemResult] = copy.deepcopy(
self._work_results)
snap_file_queue_size: int = self._file_queue_size
snap_num_running: int = len(self._in_progress)
snap_run_summarizer: BatchRunSummarizer = self._summarizer
snap_batch_id: int = self._on_batch_id
summary_json = {}
# It's uncommon that a run summarizer wouldn't be available yet but this could
# happen for example by signaling early termination to the Orchestrator.
if snap_run_summarizer:
summary_json = snap_run_summarizer.run_summary(
snap_work_results, snap_file_queue_size,
snap_num_running, self._start_time,
len(self._endpoint_managers), log_conclusion_msg)
# Write the summary json file
if write_run_summary:
try:
if self._singleton_run_summary_path:
logger.debug(
"Updating singleton run_summary: {0}".format(
self._singleton_run_summary_path))
write_json_file_atomic(
summary_json, self._singleton_run_summary_path)
else:
try:
self._status_provider.set_run_summary(
snap_batch_id, summary_json)
except BatchNotFoundException:
# This is benign and means we caught a rare race condition
# in which the batch directory is very recently deleted.
pass
# Minimal throttle on file writes. We are under _run_summary_lock.
time.sleep(3)
except Exception as e:
logger.warning("Failed to write run_summary: {0}".format(
str(e)))
if not allow_fail:
raise
def request_stop(self):
"""
Arrange for conditions that will lead to a fast conclusion
of any ongoing batch without finishing whatever is remaining or
in progress in this batch if any. This will also cause
EndpointManagers to shut down. Orchestrator's join() is
guaranteed to eventually return.
"""
# Assume this might be called from a signal handler.
# Instead of preventing child proc inheritance of signals,
# we eliminate any leaky abstractions by permitting children
# and those who spawn them to be completely blameless
# for creating unexpected conditions.
if current_process().pid != self._creator_pid:
return
with self._misc_lock:
try:
if self._config_notifier:
self._config_notifier.stop()
self._config_notifier = None
except OSError as e:
# ThreadedNotifier.stop() is not idempotent and gives
# errno EBADF if it is already stopped.
if e.errno != errno.EBADF:
raise
# A couple facts about Python3 in case there is any concern
# about being invoked by a signal handler.
# 1 - Only the main thread of a process can handle
# signals, so now we know we are the main thread of the
# creator process in the signal case.
# 2 - When running a signal handler, the main thread is
# is still subject to preemption at tick and the GIL
# can still be released for other threads. This means
# that picking up the lock here cannot create deadlock,
# unless the main thread itself was holding the lock before
# the signal. That's why we use ReentrantLock.
with self._accounting_lock:
self._stop_requested = True
while self._file_queue_size > 0:
self._file_queue.get()
self._file_queue_size -= 1
self._submission_que.put(None)
self._file_queue_cond.notify_all()
self._batch_completion_evt.set()
for m in self._endpoint_managers:
m.request_stop()
self._run_summary_thread_gate.set()
def steal_work(self, manager: EndpointManager) -> WorkItemRequest:
"""
:param manager: the EndpointManager who is trying to steal work.
:returns str: audio file of work to do
"""
sentinel = SentinelWorkItemRequest()
# Classic consumer waiter pattern using condition variable.
self._accounting_lock.acquire()
while True:
if manager.name in self._old_managers or self._stop_requested:
work = sentinel
break
if self._file_queue_size > 0:
work: WorkItemRequest = self._file_queue.get()
self._file_queue_size -= 1
# Eliminate this manager early if we detect a language mismatch.
# It will be recreated on a new batch.
if work.language and manager.endpoint_config["language"].lower(
) != work.language.lower():
self._file_queue.put(
work) # back on queue for someone qualified
self._file_queue_size += 1
self._file_queue_cond.notify()
work = sentinel
break
# Got some work to do!
self._in_progress[work.filepath] = work
self._in_progress_owner[work.filepath] = manager
break
else:
# Back to sleep because we got nothing.
self._file_queue_cond.wait(
) # implicit self.accounting_lock.release()
self._accounting_lock.release()
return work
def _merge_results(self, filepath: str, result: WorkItemResult):
if filepath not in self._work_results:
self._work_results[filepath] = result
else:
prev_attempts = self._work_results[filepath].attempts
result.attempts += prev_attempts
self._work_results[filepath] = result
def notify_work_success(self, filepath: str, manager: EndpointManager,
result: WorkItemResult):
with self._accounting_lock:
self.__cnt_work_success_cb += 1
if manager.name in self._old_managers:
# The AudioFileWork item would already be back in pending
# or running by someone else or finished. Covers an uncommon race.
return
if self._stop_requested:
# It's too late for updating batch status and we're about to die.
return
del self._in_progress[filepath]
del self._in_progress_owner[filepath]
self._merge_results(filepath, result)
# Did we just finish the batch?
if self._file_queue_size == 0 and len(self._in_progress) == 0:
self._batch_completion_evt.set()
def notify_work_failure(self, filepath: str, manager: EndpointManager,
result: WorkItemResult):
with self._accounting_lock:
self.__cnt_work_failure_cb += 1
if manager.name in self._old_managers:
# The WorkItemResult would already be back in pending
# or running by someone else or finished. Covers an uncommon race.
return
if self._stop_requested:
# It's too late for updating batch status and we're about to die.
return
self._merge_results(filepath, result)
# Do we give it another chance?
# Check retry-ability and num retries burned already.
if result.can_retry and \
self._work_results[filepath].attempts - 1 < ORCHESTRATOR_SCOPE_MAX_RETRIES:
self._log_event_que.debug(
"Placed work item {0} back into queue since retriable.".
format(filepath))
self._file_queue.put(self._in_progress[filepath])
self._file_queue_size += 1
self._file_queue_cond.notify()
# Else no more retries.
# Either way the item is no longer in progress.
del self._in_progress[filepath]
del self._in_progress_owner[filepath]
# Did we just finish the batch? E.g. finally gave up on this work
# item and that so happens to be the last in the batch.
if self._file_queue_size == 0 and len(self._in_progress) == 0:
self._batch_completion_evt.set()
def hotswap_endpoint_managers(self):
config_data = load_configuration(self._config_file,
self._strict_config)
with self._accounting_lock:
if self._stop_requested:
return
# Get the unique generation of these endpoint managers, which
# is useful for both debugging and logging.
gen = self._endpoint_generation
self._endpoint_generation += 1
# Get an EndpointStatusChecker for the type of the
# BatchRequest that is currently being processed.
ep_status_checker: EndpointStatusChecker
if isinstance(None, self._on_batch_type):
ep_status_checker = UnknownEndpointStatusChecker(
self._log_event_que)
else:
ep_status_checker = self._on_batch_type.get_endpoint_status_checker(
self._log_event_que)
try:
# Determine EndpointManagers that need to be deleted (modified, new,
# or no longer exist). Do not touch EndpointManagers that have not changed.
new_em_objs: List[EndpointManager] = []
# Start by assuming every EndpointManager needs to be deleted.
deleted_managers: Dict[str, EndpointManager] = \
{em.endpoint_name: em for em in self._endpoint_managers}
for endpoint_name, endpoint_config in config_data.items():
# If an existing endpoint is totally preserved in the new config, don't delete it.
# Also require that the endpoint's manager is not terminally stopped, otherwise we need
# a new instance of it anyway.
if endpoint_name in deleted_managers and \
endpoint_config == deleted_managers[endpoint_name].endpoint_config and \
not deleted_managers[endpoint_name]._stop_requested: # noqa
# Don't delete this EndpointManager and don't make a new one.
del deleted_managers[endpoint_name]
continue
new_em_objs.append(
EndpointManager(
"HotswapGen{0}_{1}".format(str(gen),
endpoint_name),
endpoint_name,
endpoint_config,
self._log_folder,
self._log_event_que,
self._cache_search_dirs,
# on EndpointManager has capacity to steal work
self.steal_work,
# on EndpointManager reports success
self.notify_work_success,
# on EndpointManager reports failure
self.notify_work_failure,
ep_status_checker,
))
# Validation of the config could fail or invalid yaml may have been given, etc.
# We catch anything so that we may permit another attempt later with a proper config file.
# We report it in the logs and somewhere else we will die if no forward progress for too long.
except Exception as e:
exception_details = traceback.format_exc()
logger.error(
"Caught Exception '{0}' reading config. Details: {1}\n{2}".
format(type(e).__name__, str(e), exception_details))
# Don't proceed to stop the old EndpointManagers because they're all we've got to go on.
return
if self._stop_requested:
return
# Also swap the EndpointManagers under lock in case of race.
# First stop the old EndpointManagers to be deleted.
for m in self._endpoint_managers:
if m.endpoint_name in deleted_managers:
self._old_managers.add(m.name)
m.request_stop()
# Un-assign work in progress for deleted EndpointManagers.
# Now anything the old managers might still callback
# would be rejected so we can safely move in progress back to queue.
work_in_progress = {k: v
for k, v in self._in_progress.items()
} # shallow copy
work_in_progress_owner = {
k: v
for k, v in self._in_progress_owner.items()
} # shallow copy
for filepath, work_item in self._in_progress.items():
owner_endpoint_name = self._in_progress_owner[
filepath].endpoint_name
# If the EndpointManager that owns this work item is being deleted,
# free up the work item.
if owner_endpoint_name in deleted_managers:
del work_in_progress[filepath]
del work_in_progress_owner[filepath]
self._file_queue.put(work_item)
self._file_queue_size += 1
self._in_progress = work_in_progress
self._in_progress_owner = work_in_progress_owner
# We've potentially repopulated the file_queue.
self._file_queue_cond.notify_all()
# Start the new EndpointManagers.
for m in new_em_objs:
m.start()
# Record the latest set of all EndpointManagers.
self._endpoint_managers = \
[em for em in self._endpoint_managers
if em.endpoint_name not in deleted_managers] + \
new_em_objs
# Ensure that they are all using the correct type of EndpointStatusChecker
# which depends on the subtype of BatchRequest we are currently processing.
for m in self._endpoint_managers:
m.set_endpoint_status_checker(ep_status_checker)
logger.info(
"Set new EndpointManagers after hot-swap: {0}".format(config_data))
def _master_finalize(self):
"""
Work to be done before Orchestrator's master thread exits.
"""
# Log conclusion of run_summary information if at singleton level.
if self._singleton_run_summary_path:
self.write_summary_information(write_run_summary=False,
log_conclusion_msg=True)
def _master_thread_loop(self):
# Keep doing batches until given a stop request.
while True:
# Starting a new batch.
request: BatchRequest = self._submission_que.get()
with self._accounting_lock:
self._on_batch_type = type(request)
# Recreate the endpoints on start of a new batch in case
# the last batch disabled endpoints, e.g. for mismatched
# language or other reasons.
self.hotswap_endpoint_managers()
with self._accounting_lock:
if self._stop_requested:
self._master_finalize()
return
# Starting a new batch.
# Reset record keeping if it's not singleton run summary.
if self._singleton_run_summary_path is None:
self._work_results = {}
self._summarizer = request.get_batch_run_summarizer()
logger.info("Orchestrator: Starting batch {0}".format(
request.batch_id))
self._status_provider.change_status_enum(
request.batch_id, BatchStatusEnum.running)
self._on_batch_id = request.batch_id
self._batch_completion_evt.clear()
self._run_summary_thread_gate.set()
assert len(self._in_progress) == 0
assert self._file_queue_size == 0
for work in request.make_work_items(
self._status_provider.batch_base_path(
request.batch_id), self._cache_search_dirs,
self._log_folder):
self._file_queue.put(work)
self._file_queue_size += 1
self._file_queue_cond.notify_all()
# Wait for batch completion or early stop request. In both cases,
# nothing is in progress and nothing is in queue when we're woken.
self._batch_completion_evt.wait()
logger.info("Orchestrator: Completed batch {0}".format(
request.batch_id))
# Report per-batch final run_summary.
if self._singleton_run_summary_path is None:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=True,
allow_fail=True)
# Even with singleton run_summary, we should update run_summary file
# now but not log conclusion.
else:
self.write_summary_information(write_run_summary=True,
log_conclusion_msg=False,
allow_fail=True)
# Concatenate batch-level results to single file.
if request.combine_results:
write_single_output_json(
request.files,
self._status_provider.batch_base_path(request.batch_id))
# Intentionally change status enum last so that above results committed first
# for any event-driven observers.
self._status_provider.change_status_enum(request.batch_id,
BatchStatusEnum.done)
logger.info(
"Orchestrator: Updated batch status to Done: {0}".format(
request.batch_id))
# As another batch may not show up for a while (or never), stop the periodic
# run summary thread since no new information to report.
self._run_summary_thread_gate.clear()
class IOManager(object):
def __init__(self):
self.capture_mode = False
self.child_mode = False
self.parent_mode = False
def activate_as_child(self, output_lock, output_queue, status_line_cleared):
self.parent_mode = False
self.child_mode = True
self.status_line_cleared = status_line_cleared
self.output_lock = output_lock
self.output_queue = output_queue
def activate_as_parent(self, debug=False):
assert not self.child_mode
self.debug_mode = debug
self.jobs = []
self.output_lock = Lock()
self.parent_mode = True
self.output_queue = Queue()
self.status_line_cleared = Condition()
self.thread = Thread(target=self._print_thread)
self.thread.daemon = True
self.thread.start()
def ask(self, question, default, get_input=input_function):
answers = _("[Y/n]") if default else _("[y/N]")
question = question + " " + answers + " "
with self.lock:
while True:
STDOUT_WRITER.write("\a")
STDOUT_WRITER.write(question)
STDOUT_WRITER.flush()
answer = get_input()
if answer.lower() in (_("y"), _("yes")) or (
not answer and default
):
return True
elif answer.lower() in (_("n"), _("no")) or (
not answer and not default
):
return False
STDOUT_WRITER.write(_("Please answer with 'y(es)' or 'n(o)'.\n"))
@contextmanager
def capture(self):
self.capture_mode = True
self.captured_io = {
'stderr': "",
'stdout': "",
}
yield self.captured_io
self.capture_mode = False
@property
def child_parameters(self):
return (self.output_lock, self.output_queue, self.status_line_cleared)
def debug(self, msg):
self.output_queue.put({'msg': 'LOG', 'log_type': 'DBG', 'text': msg})
def job_add(self, msg):
self.output_queue.put({'msg': 'LOG', 'log_type': 'JOB_ADD', 'text': msg})
def job_del(self, msg):
self.output_queue.put({'msg': 'LOG', 'log_type': 'JOB_DEL', 'text': msg})
def stderr(self, msg):
self.output_queue.put({'msg': 'LOG', 'log_type': 'ERR', 'text': msg})
def stdout(self, msg):
self.output_queue.put({'msg': 'LOG', 'log_type': 'OUT', 'text': msg})
@contextmanager
def job(self, job_text):
self.job_add(job_text)
yield
self.job_del(job_text)
@property
@contextmanager
def lock(self):
with self.output_lock:
self.status_line_cleared.wait()
yield
def _print_thread(self):
assert self.parent_mode
while True:
if self.output_lock.acquire(False):
msg = self.output_queue.get()
if msg['log_type'] == 'QUIT':
break
if self.debug_mode and msg['log_type'] in ('OUT', 'DBG', 'ERR'):
msg['text'] = datetime.now().strftime("[%Y-%m-%d %H:%M:%S.%f] ") + msg['text']
if self.jobs and TTY:
self._write("\r\033[K")
if msg['log_type'] == 'OUT':
self._write(msg['text'] + "\n")
elif msg['log_type'] == 'ERR':
self._write(msg['text'] + "\n", err=True)
elif msg['log_type'] == 'DBG' and self.debug_mode:
self._write(msg['text'] + "\n")
elif msg['log_type'] == 'JOB_ADD' and TTY:
self.jobs.append(msg['text'])
elif msg['log_type'] == 'JOB_DEL' and TTY:
self.jobs.remove(msg['text'])
if self.jobs and TTY:
self._write("[status] " + self.jobs[0])
self.output_lock.release()
else: # someone else is holding the output lock
# the process holding the lock should now be waiting for
# us to remove any status lines present before it starts
# printing
if self.jobs and TTY:
self._write("\r\033[K")
self.status_line_cleared.notify()
# now we wait until the other process has finished and
# released the output lock
self.output_lock.acquire()
self.output_lock.release()
def shutdown(self):
assert self.parent_mode
self.output_queue.put({'msg': 'LOG', 'log_type': 'QUIT'})
self.thread.join()
def _write(self, msg, err=False):
write_to_stream(STDERR_WRITER if err else STDOUT_WRITER, msg)
if self.capture_mode:
self.captured_io['stderr' if err else 'stdout'] += msg
class TProcessPoolServer(TServer):
"""
Server with a fixed size pool of worker subprocesses which service requests.
Note that if you need shared state between the handlers - it's up to you!
Written by Dvir Volk, doat.com
"""
def __init__(self, * args):
TServer.__init__(self, *args)
self.numWorkers = 10
self.workers = []
self.isRunning = Value('b', False)
self.stopCondition = Condition()
self.postForkCallback = None
def setPostForkCallback(self, callback):
if not callable(callback):
raise TypeError("This is not a callback!")
self.postForkCallback = callback
def setNumWorkers(self, num):
"""Set the number of worker threads that should be created"""
self.numWorkers = num
def workerProcess(self):
"""Loop around getting clients from the shared queue and process them."""
if self.postForkCallback:
self.postForkCallback()
while self.isRunning.value == True:
try:
client = self.serverTransport.accept()
self.serveClient(client)
except (KeyboardInterrupt, SystemExit):
return 0
except (Exception) as x:
logging.exception(x)
def serveClient(self, client):
"""Process input/output from a client for as long as possible"""
itrans = self.inputTransportFactory.getTransport(client)
otrans = self.outputTransportFactory.getTransport(client)
iprot = self.inputProtocolFactory.getProtocol(itrans)
oprot = self.outputProtocolFactory.getProtocol(otrans)
try:
while True:
self.processor.process(iprot, oprot)
except (TTransportException) as tx:
pass
except (Exception) as x:
logging.exception(x)
itrans.close()
otrans.close()
def serve(self):
"""Start a fixed number of worker threads and put client into a queue"""
#this is a shared state that can tell the workers to exit when set as false
self.isRunning.value = True
#first bind and listen to the port
self.serverTransport.listen()
#fork the children
for i in range(self.numWorkers):
try:
w = Process(target=self.workerProcess)
w.daemon = True
w.start()
self.workers.append(w)
except (Exception) as x:
logging.exception(x)
#wait until the condition is set by stop()
while True:
self.stopCondition.acquire()
try:
self.stopCondition.wait()
break
except (SystemExit, KeyboardInterrupt):
break
except (Exception) as x:
logging.exception(x)
self.isRunning.value = False
def stop(self):
self.isRunning.value = False
self.stopCondition.acquire()
self.stopCondition.notify()
self.stopCondition.release()
class Cpu(object):
def __init__(self):
self.pcb = None
self.__mutex = RLock()
self.__pcb_not_set = Condition(self.__mutex)
self.__mem_not_allocated = Condition(self.__mutex)
self.__round_robin_policy_on = False
def enable_round_robin(self, round_robin_quantum):
self.__round_robin_policy_on = True
self.__round_robin = RoundRobin(round_robin_quantum)
def pcb_not_set(self):
return self.__pcb_not_set
def set_kernel(self, kernel):
self.__kernel = kernel
def is_pcb_set(self):
return self.pcb != None
def set_current_pcb(self, pcb):
with self.__pcb_not_set:
self.pcb = pcb
self.__pcb_not_set.notify()
def reset_pcb(self):
self.pcb = None
def get_current_pcb(self):
return self.pcb
def __get_mem_manager(self):
return self.__kernel.get_mem_manager()
def __get_irq_manager(self):
return self.__kernel.get_irq_manager()
def fetch_decode_and_execute(self):
with self.__pcb_not_set:
while(not self.is_pcb_set()):
self.__pcb_not_set.wait()
with self.__mutex:
self.__fetch()
self.__decode()
self.__execute()
def __fetch(self):
pcb = self.get_current_pcb()
address = self.__get_mem_manager().current_instruction_address(pcb)
with self.__mem_not_allocated:
while self.__get_mem_manager().get(pcb,address) == None:
self.__mem_not_allocated.wait()
self.__current_instruction = self.__get_mem_manager().get(pcb, address )
def __decode(self):
self.__send_interruption_if_is_io()
self.__send_interruption_if_is_kill()
def __send_interruption_if_is_kill(self):
if(self.__current_instruction.is_kill_instruction()):
self.send_end()
def __send_interruption_if_is_io(self):
if(self.__current_instruction.is_io_instruction()):
self.send_io()
def __execute(self):
self.__execute_if_is_cpu_instruction()
def __execute_if_is_cpu_instruction(self):
if (self.__current_instruction.is_cpu_instruction()):
self.__current_instruction.run()
self.get_current_pcb().increment_pc()
def send_interruption(self, a_interruption):
self.__get_irq_manager().handle(Irq(a_interruption, self.get_current_pcb()))
def send_timeout(self):
self.send_interruption(TIMEOUT_INTERRUPT)
def send_end(self):
self.send_interruption(KILL_INTERRUPT)
def send_io(self):
self.send_interruption(IO_INTERRUPT)
def on_signal(self):
if self.__round_robin_policy_on:
self.__round_robin.handle_action(self)
else:
self.fetch_decode_and_execute()
