def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) if self.debug: Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( [str(p) for p in params], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath, env={str(k): str(v) for k, v in set_default(env, os.environ).items()}, shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): shell = True self.name = name self.key = (cwd, wrap(env), debug, shell) self.stdout = Queue("stdout for " + name) self.stderr = Queue("stderr for " + name) with Command.available_locker: avail = Command.available_process.setdefault(self.key, []) if not avail: self.process = Process("command shell", [cmd()], cwd, env, debug, shell, bufsize) self.process.stdin.add(set_prompt()) self.process.stdin.add("echo %errorlevel%") _wait_for_start(self.process.stdout, Null) else: self.process = avail.pop() self.process.stdin.add(" ".join(cmd_escape(p) for p in params)) self.process.stdin.add("echo %errorlevel%") self.stdout_thread = Thread.run("", self._stream_relay, self.process.stdout, self.stdout) self.stderr_thread = Thread.run("", self._stream_relay, self.process.stderr, self.stderr) self.returncode = None
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): """ Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done via proper thread-safe queues of the same name. Since the process is managed and monitored by threads, the main thread is not blocked when the child process encounters problems :param name: name given to this process :param params: list of strings for program name and parameters :param cwd: current working directory :param env: enviroment variables :param debug: true to be verbose about stdin/stdout :param shell: true to run as command line :param bufsize: if you want to screw stuff up """ self.debug = debug or DEBUG self.process_id = Process.next_process_id Process.next_process_id += 1 self.name = name + " (" + text(self.process_id) + ")" self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug) self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug) self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug) try: if cwd == None: cwd = os.getcwd() else: cwd = str(cwd) command = [str(p) for p in params] self.debug and Log.note("command: {{command}}", command=command) self.service = service = subprocess.Popen( [str(p) for p in params], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd, env={str(k): str(v) for k, v in set_default(env, os.environ).items()}, shell=shell ) self.please_stop = Signal() self.please_stop.then(self._kill) self.child_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def setup_flask_ssl(flask_app, flask_config): """ SPAWN A NEW THREAD TO RUN AN SSL ENDPOINT REMOVES ssl_context FROM flask_config BEFORE RETURNING :param flask_app: :param flask_config: :return: """ if not flask_config.ssl_context: return ssl_flask = flask_config.copy() ssl_flask.debug = False ssl_flask.port = 443 if is_data(flask_config.ssl_context): # EXPECTED PEM ENCODED FILE NAMES # `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS with TempFile() as tempfile: try: tempfile.write( File(ssl_flask.ssl_context.certificate_file).read_bytes() ) if ssl_flask.ssl_context.certificate_chain_file: tempfile.write( File(ssl_flask.ssl_context.certificate_chain_file).read_bytes() ) tempfile.flush() tempfile.close() context = SSLContext(PROTOCOL_SSLv23) context.load_cert_chain( tempfile.name, keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath, ) ssl_flask.ssl_context = context except Exception as e: Log.error("Could not handle ssl context construction", cause=e) def runner(please_stop): Log.warning( "ActiveData listening on encrypted port {{port}}", port=ssl_flask.port ) flask_app.run(**ssl_flask) Thread.run("SSL Server", runner) if flask_config.ssl_context and flask_config.port != 80: Log.warning( "ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}", port=flask_config.port, ) flask_config.ssl_context = None
def __init__( self, name, slow_queue, # THE SLOWER QUEUE batch_size=None, # THE MAX SIZE OF BATCHES SENT TO THE SLOW QUEUE max_size=None, # SET THE MAXIMUM SIZE OF THE QUEUE, WRITERS WILL BLOCK IF QUEUE IS OVER THIS LIMIT period=None, # MAX TIME (IN SECONDS) BETWEEN FLUSHES TO SLOWER QUEUE silent=False, # WRITES WILL COMPLAIN IF THEY ARE WAITING TOO LONG error_target=None # CALL error_target(error, buffer) **buffer IS THE LIST OF OBJECTS ATTEMPTED** # BE CAREFUL! THE THREAD MAKING THE CALL WILL NOT BE YOUR OWN! # DEFAULT BEHAVIOUR: THIS WILL KEEP RETRYING WITH WARNINGS ): if period != None and not isinstance(period, (int, float, long)): Log.error("Expecting a float for the period") period = coalesce(period, 1) # SECONDS batch_size = coalesce(batch_size, int(max_size / 2) if max_size else None, 900) max_size = coalesce(max_size, batch_size * 2) # REASONABLE DEFAULT Queue.__init__(self, name=name, max=max_size, silent=silent) self.name = name self.slow_queue = slow_queue self.thread = Thread.run("threaded queue for " + name, self.worker_bee, batch_size, period, error_target) # parent_thread=self)
def __init__(self, flask_app, db, cookie, table="sessions"): global SINGLTON if SINGLTON: Log.error("Can only handle one session manager at a time") SINGLTON = self if is_data(db): self.db = Sqlite(db) else: self.db = db self.table = table self.cookie = cookie self.cookie.max_lifetime = parse(self.cookie.max_lifetime) self.cookie.inactive_lifetime = parse(self.cookie.inactive_lifetime) if not self.db.about(self.table): self.setup() Thread.run("session monitor", self.monitor)
from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from mo_logs import Log from mo_threads.lock import Lock from mo_threads.multiprocess import Process from mo_threads.queues import Queue, ThreadedQueue from mo_threads.signal import Signal from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD from mo_threads.till import Till Log.cprofiler_stats = Queue( "cprofiler stats") # ACCUMULATION OF STATS FROM ALL THREADS MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon) MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers) # from threading import Thread as _threading_Thread # _temp = _threading_Thread.setDaemon # # fixes = [] # # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR # try: # from paramiko import Transport # # def fix(self): # if isinstance(self, Transport): # self.stop = self.close # WE KNOW Transport DOES NOT HAVE A stop() METHOD, SO ADDING SHOULD BE FINE # parent = Thread.current() # parent.add_child(self)
def __init__( self, name, queue, # THE SLOWER QUEUE batch_size=None, # THE MAX SIZE OF BATCHES SENT TO THE SLOW QUEUE max_size=None, # SET THE MAXIMUM SIZE OF THE QUEUE, WRITERS WILL BLOCK IF QUEUE IS OVER THIS LIMIT period=None, # MAX TIME (IN SECONDS) BETWEEN FLUSHES TO SLOWER QUEUE silent=False, # WRITES WILL COMPLAIN IF THEY ARE WAITING TOO LONG error_target=None # CALL THIS WITH ERROR **AND THE LIST OF OBJECTS ATTEMPTED** # BE CAREFUL! THE THREAD MAKING THE CALL WILL NOT BE YOUR OWN! # DEFAULT BEHAVIOUR: THIS WILL KEEP RETRYING WITH WARNINGS ): if period != None and not isinstance(period, (int, float, long)): Log.error("Expecting a float for the period") batch_size = coalesce(batch_size, int(max_size / 2) if max_size else None, 900) max_size = coalesce(max_size, batch_size * 2) # REASONABLE DEFAULT period = coalesce(period, 1) # SECONDS Queue.__init__(self, name=name, max=max_size, silent=silent) def worker_bee(please_stop): please_stop.on_go(lambda: self.add(THREAD_STOP)) _buffer = [] _post_push_functions = [] now = time() next_push = Till(till=now + period) # THE TIME WE SHOULD DO A PUSH last_push = now - period def push_to_queue(): queue.extend(_buffer) del _buffer[:] for ppf in _post_push_functions: ppf() del _post_push_functions[:] while not please_stop: try: if not _buffer: item = self.pop() now = time() if now > last_push + period: # Log.note("delay next push") next_push = Till(till=now + period) else: item = self.pop(till=next_push) now = time() if item is THREAD_STOP: push_to_queue() please_stop.go() break elif isinstance(item, types.FunctionType): _post_push_functions.append(item) elif item is not None: _buffer.append(item) except Exception as e: e = Except.wrap(e) if error_target: try: error_target(e, _buffer) except Exception as f: Log.warning( "`error_target` should not throw, just deal", name=name, cause=f) else: Log.warning("Unexpected problem", name=name, cause=e) try: if len(_buffer) >= batch_size or next_push: if _buffer: push_to_queue() last_push = now = time() next_push = Till(till=now + period) except Exception as e: e = Except.wrap(e) if error_target: try: error_target(e, _buffer) except Exception as f: Log.warning( "`error_target` should not throw, just deal", name=name, cause=f) else: Log.warning( "Problem with {{name}} pushing {{num}} items to data sink", name=name, num=len(_buffer), cause=e) if _buffer: # ONE LAST PUSH, DO NOT HAVE TIME TO DEAL WITH ERRORS push_to_queue() self.thread = Thread.run("threaded queue for " + name, worker_bee) # parent_thread=self)
# THIS THREADING MODULE IS PERMEATED BY THE please_stop SIGNAL. # THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS # FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from mo_threads.lock import Lock from mo_threads.multiprocess import Process from mo_threads.queues import Queue, ThreadedQueue from mo_threads.signal import Signal from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD from mo_threads.till import Till MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon) MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers) # from threading import Thread as _threading_Thread # _temp = _threading_Thread.setDaemon # # fixes = [] # # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR # try: # from paramiko import Transport # # def fix(self):
"`error_target` should not throw, just deal", name=name, cause=f) else: _Log.warning( "Problem with {{name}} pushing {{num}} items to data sink", name=name, num=len(_buffer), cause=e) if _buffer: # ONE LAST PUSH, DO NOT HAVE TIME TO DEAL WITH ERRORS push_to_queue() self.thread = Thread.run("threaded queue for " + name, worker_bee, parent_thread=self) def add(self, value, timeout=None): with self.lock: self._wait_for_queue_space(timeout=timeout) if not self.please_stop: self.queue.append(value) # if Random.range(0, 50) == 0: # sizes = wrap([{"id":i["id"], "size":len(convert.value2json(i))} for i in self.queue if isinstance(i, Mapping)]) # size=sum(sizes.size) # if size>50000000: # from pyLibrary.queries import jx # # biggest = jx.sort(sizes, "size").last().id # _Log.note("Big record {{id}}", id=biggest)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + string2quote(name)) self.stdin = Queue("stdin for process " + string2quote(name), silent=True) self.stdout = Queue("stdout for process " + string2quote(name), silent=True) self.stderr = Queue("stderr for process " + string2quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception, e: Log.error("Can not call", e)