def main(workers=10):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
start_logging()
log_info("New multiprocessing session with {} workers".format(workers))
# Input JoinableQueue and Output Queue
inq = JoinableQueue(maxsize=int(workers * 1.5))
outq = Queue(maxsize=int(workers * 1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
# Feed the process pool with work units
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
# which terminates the process pool
for _ in range(workers):
inq.put(None)
inq.join()
print("Control process terminating")
def test_console_output_and_time(self):
" Tests the resulting console output from the framework's output. "
regex_workers = re.compile("(Worker Thread-[\d]+ done[\n]){10}")
saved_output = sys.stdout
try:
out = StringIO()
sys.stdout = out
control.main()
output = out.getvalue().strip()
# Due to random nature, separate the presence of console output
self.assertTrue(regex_workers.match(output))
self.assertTrue(re.search("Final length of random string: 1000", output))
self.assertTrue(re.search("Control thread terminating", output))
self.assertTrue(re.search("Output thread terminating", output))
# Now that console output is nullified, log total time
timer = Timer("t = control.main()", "from __main__ import control")
log_info("Latest run took {} secs to complete".format(timer.timeit(number=10)))
finally:
sys.stdout = saved_output
def test_console_output_and_time(self):
" Tests the resulting console output from the framework's output. "
regex_workers = re.compile("(Worker Thread-[\d]+ done[\n]){10}")
saved_output = sys.stdout
try:
out = StringIO()
sys.stdout = out
control.main()
output = out.getvalue().strip()
# Due to random nature, separate the presence of console output
self.assertTrue(regex_workers.match(output))
self.assertTrue(
re.search("Final length of random string: 1000", output))
self.assertTrue(re.search("Control thread terminating", output))
self.assertTrue(re.search("Output thread terminating", output))
# Now that console output is nullified, log total time
timer = Timer("t = control.main()", "from __main__ import control")
log_info("Latest run took {} secs to complete".format(
timer.timeit(number=10)))
finally:
sys.stdout = saved_output
def main(workers=10):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
start_logging()
log_info("New multiprocessing session with {} workers".format(workers))
# Input JoinableQueue and Output Queue
inq = JoinableQueue(maxsize=int(workers*1.5))
outq = Queue(maxsize=int(workers*1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
# Feed the process pool with work units
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
# which terminates the process pool
for _ in range(workers):
inq.put(None)
inq.join()
print("Control process terminating")
def main(workers=WORKERS):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
log_info("New mini-framework session with {} workers".format(workers))
inq = Queue(maxsize=int(workers * 1.5))
outq = Queue(maxsize=int(workers * 1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
for _ in range(workers):
inq.put(None)
inq.join()
print("Control thread terminating")
log_info("Mini-framework finished. Len: {} chars".format(len(ot.output)))
def __exit__(self, exc_type, exc_value, exc_tb):
log_info("A {} has been detected".format(exc_type))
# Run through the exception list received
for self_exc in self.exceptions:
if isinstance(exc_type, self_exc):
log_info("Avoiding {} to be raised".format(exc_type))
return False
return True
def __exit__(self, exc_type, exc_value, exc_tb):
log_info("A {} has been detected".format(exc_type))
# Run through the exception list received
for self_exc in self.exceptions:
if isinstance(exc_type, self_exc):
log_info("Avoiding {} to be raised".format(exc_type))
return False
return True
def main():
start_logging()
for chunk_size in 1000, 10000, 100000, 1000000:
w = Timer("file_writer({})".format(chunk_size),
"from __main__ import file_writer")
mm = Timer("mmap_file_writer({})".format(chunk_size),
"from __main__ import mmap_file_writer")
wt = "file_writer took {} w/chunks of {}".format(
w.timeit(number=1), chunk_size)
mt = "mmap_file_writer took {} w/chunks of {}".format(
mm.timeit(number=1), chunk_size)
print(wt)
print(mt)
log_info(wt)
log_info(mt)
def main():
start_logging()
for chunk_size in 1000, 10000, 100000, 1000000:
w = Timer("file_writer({})".format(chunk_size),
"from __main__ import file_writer")
mm = Timer("mmap_file_writer({})".format(chunk_size),
"from __main__ import mmap_file_writer")
wt = "file_writer took {} w/chunks of {}".format(w.timeit(number=1),
chunk_size)
mt = "mmap_file_writer took {} w/chunks of {}".format(mm.timeit(number=1),
chunk_size)
print(wt)
print(mt)
log_info(wt)
log_info(mt)
def run(self):
" Extract items from the output queue and print until all done. "
while self.workers:
p = self.queue.get()
if p is None:
self.workers -= 1
else:
# This is a real output packet
self.output.append(p)
start_logging()
output_txt = "Final length of random string: {}".format(len(self.output))
print(output_txt)
log_info(output_txt)
print("Output process terminating")
sys.stdout.flush()
def run(self):
" Extract items from the output queue and print until all done. "
while self.workers:
p = self.queue.get()
if p is None:
self.workers -= 1
else:
# This is a real output packet
self.output.append(p)
start_logging()
output_txt = "Final length of random string: {}".format(
len(self.output))
print(output_txt)
log_info(output_txt)
print("Output process terminating")
sys.stdout.flush()
def __init__(self, *exceptions):
" Receives Exception classes that will be allowed to be raised. "
self.exceptions = list(exceptions)
log_info("New CM_AvoidExceptions against {}".format(self.exceptions))
def __init__(self, *exceptions):
" Receives Exception classes that will be allowed to be raised. "
self.exceptions = list(exceptions)
log_info("New CM_AvoidExceptions against {}".format(self.exceptions))