def check_Worker3():
from pulo import random_string
def print_random_string(n):
str = random_string(n)
Synched.out("print_random_string:: {rnd}", rnd = str)
def print_string(fmt, *args, **kw):
Synched.out(fmt, *args, **kw)
def decorate_string(string):
string = "<<" + string + ">>"
producer = QueueWorker("producer")
try:
while True:
data = raw_input("Enter some data> ")
Synched.out("value? {value}; type? {type}", value = data, type = type(data))
producer.quit()
producer.join()
#t1 = Task("random_string", print_random_string, 17)
except EOFError:
print "Quitting"
def transform_str(e_exit, e_pause, e_wake, q_input, q_output, f_mapitem):
self = threading.current_thread()
e_any = OrEvent(e_exit, e_pause, e_wake)
def transform():
try:
item = q_input.get_nowait()
q_input.task_done()
result = f_mapitem(item)
q_output.put(result)
Synched.out("{name} transformed << {prod} | {trans}>>",
name=str(self.name),
prod=str(item),trans = result)
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
while True:
if e_exit.is_set():
break
transform()
Synched.out("{name} exiting", name = str(self.name))
def consume_str(e_exit, e_pause, e_wake, q_output, f_consume):#, q_output):
self = threading.current_thread()
e_any = OrEvent(e_exit, e_pause, e_wake)
def consume():
try:
item = q_output.get_nowait()
Synched.out("{name} consumed >> {prod} <<", name=str(self.name),
prod=str(item))
q_output.task_done()
#q_output.put(item)
ret = f_consume(item)
if False == ret:
return False
else:
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
while True:
if e_exit.is_set():
break
consume()
Synched.out("{name} exiting", name = str(self.name))
def transform():
try:
item = q_input.get_nowait()
q_input.task_done()
result = f_mapitem(item)
q_output.put(result)
Synched.out("{name} transformed << {prod} | {trans}>>",
name=str(self.name),
prod=str(item),trans = result)
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
def thread_say(fmt, *args, **keys):
_name = threading.current_thread().name
label_string = "Thread:{name}".format(name=_name)
fmt_string = fmt.format(*args, **keys)
full_string = "{label}:: {fmt}".format(label=label_string, fmt=fmt_string)
return Synched.out(full_string)
def consume():
try:
item = q_output.get_nowait()
Synched.out("{name} consumed >> {prod} <<", name=str(self.name),
prod=str(item))
q_output.task_done()
#q_output.put(item)
ret = f_consume(item)
if False == ret:
return False
else:
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
def produce_str(e_exit, e_pause, e_wake, q_input, N):
self = threading.current_thread()
def produce():
string = random_string(N)
Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string)
q_input.put(string)
time.sleep(1)
while True:
if e_exit.is_set():
break
if e_pause.is_set():
e_wake.wait()
if e_wake.is_set():
e_wake.clear()
e_pause.clear()
produce()
Synched.out("{name} exiting", name = str(self.name))
def pretty_print(item, label):
Synched.out("-=@ {label}: {item} @=-", label=label, item=item)
def check_Worker2():
from pulo import random_string
import random
counts = {'producers': 5, 'consumers': 10}
indices = {'producers': 0, 'consumers': 1}
names = ["Producer[%d]", "Consumer[%d]"]
def print_thread(fmt, *args, **kw):
name = threading.current_thread().name
fmt = name + ": " + fmt
return Synched.out(fmt, *args, **kw)
def produce_number(min_, max_):
n = random.randint(min_, max_)
print_thread("produce_number: {min_} < {n} < {max_}", min_ = min_, n = n,
max_ = max_)
return n
def produce_str(N):
s = random_string(N)
print_thread("produce_str: <{s}>; N = {N}", s = s, N = N)
return s
def decorate_str(str):
ret = "-==@ " + str + " @==-"
print_thread("decorate_str: <{ret}>", ret = ret)
return ret
def pretty_print(str):
print_thread("{{ {str} }}", str = str)
def delay(seconds):
print_thread("delaying for {sec}s", sec=seconds)
time.sleep(seconds)
task_num = Task(produce_number, 5, 25)
task_str = Task(produce_str, N = task_num)
task_str.link(Task(delay), 2)# .link(Task(delay), 2)
#print "task_num():", task_num()
#print "task_str():", task_str()
e_quit = threading.Event()
q_tasks = Queue()
threads = []
def create_threads(which, queue):
for x in xrange(1, counts[which]):
name = names[indices[which]] % x
thread = QueueWorker(queue, e_quit, name=name)
threads.append(thread)
create_threads('producers', q_tasks)
start_all(threads)
Synched.out("started")
try:
while True:
inp = raw_input("> ")
if inp == "y":
break
elif inp == "a":
for x in xrange(''):
pass
q_tasks.put(task_str)
except EOFError, err:
pass
def produce():
string = random_string(N)
Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string)
q_input.put(string)
time.sleep(1)
def pretty_print(str):
Synched.out("[[ {str} ]]", str = str)
def check_Threads():
import time
from pulo import random_string
def decorate_str(str):
return "-==@ " + str + " @==-"
def pretty_print(str):
Synched.out("[[ {str} ]]", str = str)
threads = []
producers = []
counts = { 'producers': 2, 'consumers' : 2, 'transformers': 5}
event_exit = threading.Event()
event_pause = threading.Event()
event_wake = threading.Event()
queue_input = Queue()
queue_output = Queue()
#queue_filtered = Queue()
#
# Producer target
#
def produce_str(e_exit, e_pause, e_wake, q_input, N):
self = threading.current_thread()
def produce():
string = random_string(N)
Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string)
q_input.put(string)
time.sleep(1)
while True:
if e_exit.is_set():
break
if e_pause.is_set():
e_wake.wait()
if e_wake.is_set():
e_wake.clear()
e_pause.clear()
produce()
Synched.out("{name} exiting", name = str(self.name))
#
# Producers
#
for k in xrange(0, counts['producers']):
n = 10+k
thread = threading.Thread(
target=produce_str,
name = ("Producer[%d]" % k),
args=(event_exit, event_pause, event_wake, queue_input, n)
)
threads.append(thread)
producers = threads
#
# Transformer target
#
def transform_str(e_exit, e_pause, e_wake, q_input, q_output, f_mapitem):
self = threading.current_thread()
e_any = OrEvent(e_exit, e_pause, e_wake)
def transform():
try:
item = q_input.get_nowait()
q_input.task_done()
result = f_mapitem(item)
q_output.put(result)
Synched.out("{name} transformed << {prod} | {trans}>>",
name=str(self.name),
prod=str(item),trans = result)
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
while True:
if e_exit.is_set():
break
transform()
Synched.out("{name} exiting", name = str(self.name))
#
# Transformers
#
for k in xrange(0, counts['transformers']):
thread = threading.Thread(
target=transform_str,
name = ("Transformer[%d]" % k),
args=(event_exit, event_pause, event_wake, queue_input,
queue_output, decorate_str)
)
threads.append(thread)
#
# Consumer target
#
def consume_str(e_exit, e_pause, e_wake, q_output, f_consume):#, q_output):
self = threading.current_thread()
e_any = OrEvent(e_exit, e_pause, e_wake)
def consume():
try:
item = q_output.get_nowait()
Synched.out("{name} consumed >> {prod} <<", name=str(self.name),
prod=str(item))
q_output.task_done()
#q_output.put(item)
ret = f_consume(item)
if False == ret:
return False
else:
return True
except Empty:
Synched.out("{name} queue empty, sleeping...",
name=str(self.name))
time.sleep(1)
return False
while True:
if e_exit.is_set():
break
consume()
Synched.out("{name} exiting", name = str(self.name))
#
# Consumers
#
for k in xrange(0, counts['consumers']):
n = 10+k
thread = threading.Thread(
target=consume_str,
name = ("Consumer[%d]" % k),
args=(event_exit, event_pause, event_wake, queue_output,
pretty_print)#, queue_output)
)
threads.append(thread)
start_all(threads)
data = None
try:
while True:
data = raw_input("Quit? [y/n] ")
if data == "y" or data == "Y":
break
elif data == "s" or data == "S":
Synched.out("Putting them to sleep for 10 secs.")
event_pause.set()
time.sleep(10)
event_wake.set()
except EOFError:
pass
event_exit.set()
join_all(threads)
def empty_queue(q, name):
print ""
print "%s: [" % str(name)
while False == q.empty():
item = q.get()
print "%s" % item
print "]"
empty_queue(queue_input, "queue_input")
#empty_queue(queue_filtered, "queue_filtered")
empty_queue(queue_output, "queue_output")
def queue_empty(q, name):
Synched.out("")
thread_say("queue {name}::", name=name)
while not q.empty():
item = q.get()
thread_say("\titem: {item}", item = item)
def empty_queue():
Synched.out("")
thread_say("queue::")
while not queue.empty():
item = queue.get()
thread_say("\titem: {item}", item = item)
def print_string(fmt, *args, **kw):
Synched.out(fmt, *args, **kw)
def print_random_string(n):
str = random_string(n)
Synched.out("print_random_string:: {rnd}", rnd = str)
def print_thread(fmt, *args, **kw):
name = threading.current_thread().name
fmt = name + ": " + fmt
return Synched.out(fmt, *args, **kw)
