class Multithread(object): """ SIMPLE SEMANTICS FOR SYMMETRIC MULTITHREADING PASS A SET OF functions TO BE EXECUTED (ONE PER THREAD) SET outbound==False TO SIMPLY THROW AWAY RETURN VALUES, IF ANY threads - IF functions IS NOT AN ARRAY, THEN threads IS USED TO MAKE AN ARRAY THE inbound QUEUE IS EXPECTING dicts, EACH dict IS USED AS kwargs TO GIVEN functions """ def __init__(self, functions, threads=None, outbound=None, silent_queues=None): if outbound is None: self.outbound = Queue("multithread", silent=silent_queues) elif outbound is False: self.outbound = None else: self.outbound = outbound self.inbound = Queue("multithread", silent=silent_queues) # MAKE THREADS if isinstance(functions, Iterable): Log.error("Not supported anymore") self.threads = [] for t in range(coalesce(threads, 1)): thread = worker_thread("worker " + unicode(t), self.inbound, self.outbound, functions) self.threads.append(thread) def __enter__(self): return self # WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING def __exit__(self, type, value, traceback): try: if isinstance(value, Exception): self.inbound.close() for t in self.threads: t.keep_running = False else: # ADD STOP MESSAGE, ONE FOR EACH THREAD, FOR ORDERLY SHUTDOWN for t in self.threads: self.inbound.add(Thread.STOP) self.join() except Exception, e: Log.warning("Problem sending stops", e)
class Log_usingQueue(BaseLog): def __init__(self): self.queue = Queue("log messages") def write(self, template, params): self.queue.add(expand_template(template, params)) def stop(self): self.queue.close() def pop(self): lines = self.queue.pop() output = [] for l in lines.split("\n"): if l[19:22] == " - ": l = l[22:] if l.strip().startswith("File"): continue output.append(l) return "\n".join(output).strip()
class TextLog_usingQueue(TextLog): def __init__(self, name=None): queue_name = "log messages to queue" if name: queue_name += " "+name self.queue = Queue(queue_name) def write(self, template, params): self.queue.add(expand_template(template, params)) def stop(self): self.queue.close() def pop(self): lines = self.queue.pop() output = [] for l in lines.split("\n"): if l[19:22] == " - ": l = l[22:] if l.strip().startswith("File"): continue output.append(l) return "\n".join(output).strip()
class Multiprocess(object): # THE COMPLICATION HERE IS CONNECTING THE DISPARATE LOGGING TO # A CENTRAL POINT # ONLY THE MAIN THREAD CAN CREATE AND COMMUNICATE WITH multiprocess.Process def __init__(self, functions): self.outbound = Queue("out to process") self.inbound = Queue("in from stdin") self.inbound = Queue("in from stderr") # MAKE # MAKE THREADS self.threads = [] for t, f in enumerate(functions): thread = worker( "worker " + unicode(t), f, self.inbound, self.outbound, ) self.threads.append(thread) def __enter__(self): return self # WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING def __exit__(self, a, b, c): try: self.inbound.close() # SEND STOPS TO WAKE UP THE WORKERS WAITING ON inbound.pop() except Exception, e: Log.warning("Problem adding to inbound", e) self.join()