def check_Worker3(): from pulo import random_string def print_random_string(n): str = random_string(n) Synched.out("print_random_string:: {rnd}", rnd = str) def print_string(fmt, *args, **kw): Synched.out(fmt, *args, **kw) def decorate_string(string): string = "<<" + string + ">>" producer = QueueWorker("producer") try: while True: data = raw_input("Enter some data> ") Synched.out("value? {value}; type? {type}", value = data, type = type(data)) producer.quit() producer.join() #t1 = Task("random_string", print_random_string, 17) except EOFError: print "Quitting"
def transform_str(e_exit, e_pause, e_wake, q_input, q_output, f_mapitem): self = threading.current_thread() e_any = OrEvent(e_exit, e_pause, e_wake) def transform(): try: item = q_input.get_nowait() q_input.task_done() result = f_mapitem(item) q_output.put(result) Synched.out("{name} transformed << {prod} | {trans}>>", name=str(self.name), prod=str(item),trans = result) return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False while True: if e_exit.is_set(): break transform() Synched.out("{name} exiting", name = str(self.name))
def consume_str(e_exit, e_pause, e_wake, q_output, f_consume):#, q_output): self = threading.current_thread() e_any = OrEvent(e_exit, e_pause, e_wake) def consume(): try: item = q_output.get_nowait() Synched.out("{name} consumed >> {prod} <<", name=str(self.name), prod=str(item)) q_output.task_done() #q_output.put(item) ret = f_consume(item) if False == ret: return False else: return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False while True: if e_exit.is_set(): break consume() Synched.out("{name} exiting", name = str(self.name))
def transform(): try: item = q_input.get_nowait() q_input.task_done() result = f_mapitem(item) q_output.put(result) Synched.out("{name} transformed << {prod} | {trans}>>", name=str(self.name), prod=str(item),trans = result) return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False
def thread_say(fmt, *args, **keys): _name = threading.current_thread().name label_string = "Thread:{name}".format(name=_name) fmt_string = fmt.format(*args, **keys) full_string = "{label}:: {fmt}".format(label=label_string, fmt=fmt_string) return Synched.out(full_string)
def consume(): try: item = q_output.get_nowait() Synched.out("{name} consumed >> {prod} <<", name=str(self.name), prod=str(item)) q_output.task_done() #q_output.put(item) ret = f_consume(item) if False == ret: return False else: return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False
def produce_str(e_exit, e_pause, e_wake, q_input, N): self = threading.current_thread() def produce(): string = random_string(N) Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string) q_input.put(string) time.sleep(1) while True: if e_exit.is_set(): break if e_pause.is_set(): e_wake.wait() if e_wake.is_set(): e_wake.clear() e_pause.clear() produce() Synched.out("{name} exiting", name = str(self.name))
def pretty_print(item, label): Synched.out("-=@ {label}: {item} @=-", label=label, item=item)
def check_Worker2(): from pulo import random_string import random counts = {'producers': 5, 'consumers': 10} indices = {'producers': 0, 'consumers': 1} names = ["Producer[%d]", "Consumer[%d]"] def print_thread(fmt, *args, **kw): name = threading.current_thread().name fmt = name + ": " + fmt return Synched.out(fmt, *args, **kw) def produce_number(min_, max_): n = random.randint(min_, max_) print_thread("produce_number: {min_} < {n} < {max_}", min_ = min_, n = n, max_ = max_) return n def produce_str(N): s = random_string(N) print_thread("produce_str: <{s}>; N = {N}", s = s, N = N) return s def decorate_str(str): ret = "-==@ " + str + " @==-" print_thread("decorate_str: <{ret}>", ret = ret) return ret def pretty_print(str): print_thread("{{ {str} }}", str = str) def delay(seconds): print_thread("delaying for {sec}s", sec=seconds) time.sleep(seconds) task_num = Task(produce_number, 5, 25) task_str = Task(produce_str, N = task_num) task_str.link(Task(delay), 2)# .link(Task(delay), 2) #print "task_num():", task_num() #print "task_str():", task_str() e_quit = threading.Event() q_tasks = Queue() threads = [] def create_threads(which, queue): for x in xrange(1, counts[which]): name = names[indices[which]] % x thread = QueueWorker(queue, e_quit, name=name) threads.append(thread) create_threads('producers', q_tasks) start_all(threads) Synched.out("started") try: while True: inp = raw_input("> ") if inp == "y": break elif inp == "a": for x in xrange(''): pass q_tasks.put(task_str) except EOFError, err: pass
def produce(): string = random_string(N) Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string) q_input.put(string) time.sleep(1)
def pretty_print(str): Synched.out("[[ {str} ]]", str = str)
def check_Threads(): import time from pulo import random_string def decorate_str(str): return "-==@ " + str + " @==-" def pretty_print(str): Synched.out("[[ {str} ]]", str = str) threads = [] producers = [] counts = { 'producers': 2, 'consumers' : 2, 'transformers': 5} event_exit = threading.Event() event_pause = threading.Event() event_wake = threading.Event() queue_input = Queue() queue_output = Queue() #queue_filtered = Queue() # # Producer target # def produce_str(e_exit, e_pause, e_wake, q_input, N): self = threading.current_thread() def produce(): string = random_string(N) Synched.out("{name} produced << {prod} >>", name=str(self.name), prod=string) q_input.put(string) time.sleep(1) while True: if e_exit.is_set(): break if e_pause.is_set(): e_wake.wait() if e_wake.is_set(): e_wake.clear() e_pause.clear() produce() Synched.out("{name} exiting", name = str(self.name)) # # Producers # for k in xrange(0, counts['producers']): n = 10+k thread = threading.Thread( target=produce_str, name = ("Producer[%d]" % k), args=(event_exit, event_pause, event_wake, queue_input, n) ) threads.append(thread) producers = threads # # Transformer target # def transform_str(e_exit, e_pause, e_wake, q_input, q_output, f_mapitem): self = threading.current_thread() e_any = OrEvent(e_exit, e_pause, e_wake) def transform(): try: item = q_input.get_nowait() q_input.task_done() result = f_mapitem(item) q_output.put(result) Synched.out("{name} transformed << {prod} | {trans}>>", name=str(self.name), prod=str(item),trans = result) return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False while True: if e_exit.is_set(): break transform() Synched.out("{name} exiting", name = str(self.name)) # # Transformers # for k in xrange(0, counts['transformers']): thread = threading.Thread( target=transform_str, name = ("Transformer[%d]" % k), args=(event_exit, event_pause, event_wake, queue_input, queue_output, decorate_str) ) threads.append(thread) # # Consumer target # def consume_str(e_exit, e_pause, e_wake, q_output, f_consume):#, q_output): self = threading.current_thread() e_any = OrEvent(e_exit, e_pause, e_wake) def consume(): try: item = q_output.get_nowait() Synched.out("{name} consumed >> {prod} <<", name=str(self.name), prod=str(item)) q_output.task_done() #q_output.put(item) ret = f_consume(item) if False == ret: return False else: return True except Empty: Synched.out("{name} queue empty, sleeping...", name=str(self.name)) time.sleep(1) return False while True: if e_exit.is_set(): break consume() Synched.out("{name} exiting", name = str(self.name)) # # Consumers # for k in xrange(0, counts['consumers']): n = 10+k thread = threading.Thread( target=consume_str, name = ("Consumer[%d]" % k), args=(event_exit, event_pause, event_wake, queue_output, pretty_print)#, queue_output) ) threads.append(thread) start_all(threads) data = None try: while True: data = raw_input("Quit? [y/n] ") if data == "y" or data == "Y": break elif data == "s" or data == "S": Synched.out("Putting them to sleep for 10 secs.") event_pause.set() time.sleep(10) event_wake.set() except EOFError: pass event_exit.set() join_all(threads) def empty_queue(q, name): print "" print "%s: [" % str(name) while False == q.empty(): item = q.get() print "%s" % item print "]" empty_queue(queue_input, "queue_input") #empty_queue(queue_filtered, "queue_filtered") empty_queue(queue_output, "queue_output")
def queue_empty(q, name): Synched.out("") thread_say("queue {name}::", name=name) while not q.empty(): item = q.get() thread_say("\titem: {item}", item = item)
def empty_queue(): Synched.out("") thread_say("queue::") while not queue.empty(): item = queue.get() thread_say("\titem: {item}", item = item)
def print_string(fmt, *args, **kw): Synched.out(fmt, *args, **kw)
def print_random_string(n): str = random_string(n) Synched.out("print_random_string:: {rnd}", rnd = str)
def print_thread(fmt, *args, **kw): name = threading.current_thread().name fmt = name + ": " + fmt return Synched.out(fmt, *args, **kw)