def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
shell = True
self.name = name
self.key = (cwd, wrap(env), debug, shell)
self.stdout = Queue("stdout for " + name)
self.stderr = Queue("stderr for " + name)
with Command.available_locker:
avail = Command.available_process.setdefault(self.key, [])
if not avail:
self.process = Process("command shell", [cmd()], cwd, env,
debug, shell, bufsize)
self.process.stdin.add(set_prompt())
self.process.stdin.add("echo %errorlevel%")
_wait_for_start(self.process.stdout, Null)
else:
self.process = avail.pop()
self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
self.process.stdin.add("echo %errorlevel%")
self.stdout_thread = Thread.run("", self._stream_relay,
self.process.stdout, self.stdout)
self.stderr_thread = Thread.run("", self._stream_relay,
self.process.stderr, self.stderr)
self.returncode = None
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
"""
Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
via proper thread-safe queues of the same name.
Since the process is managed and monitored by threads, the main thread is not blocked when the child process
encounters problems
:param name: name given to this process
:param params: list of strings for program name and parameters
:param cwd: current working directory
:param env: enviroment variables
:param debug: true to be verbose about stdin/stdout
:param shell: true to run as command line
:param bufsize: if you want to screw stuff up
"""
self.debug = debug or DEBUG
self.process_id = Process.next_process_id
Process.next_process_id += 1
self.name = name + " (" + text(self.process_id) + ")"
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
command = [str(p) for p in params]
self.debug and Log.note("command: {{command}}", command=command)
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def setup_flask_ssl(flask_app, flask_config):
"""
SPAWN A NEW THREAD TO RUN AN SSL ENDPOINT
REMOVES ssl_context FROM flask_config BEFORE RETURNING
:param flask_app:
:param flask_config:
:return:
"""
if not flask_config.ssl_context:
return
ssl_flask = flask_config.copy()
ssl_flask.debug = False
ssl_flask.port = 443
if is_data(flask_config.ssl_context):
# EXPECTED PEM ENCODED FILE NAMES
# `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS
with TempFile() as tempfile:
try:
tempfile.write(
File(ssl_flask.ssl_context.certificate_file).read_bytes()
)
if ssl_flask.ssl_context.certificate_chain_file:
tempfile.write(
File(ssl_flask.ssl_context.certificate_chain_file).read_bytes()
)
tempfile.flush()
tempfile.close()
context = SSLContext(PROTOCOL_SSLv23)
context.load_cert_chain(
tempfile.name,
keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath,
)
ssl_flask.ssl_context = context
except Exception as e:
Log.error("Could not handle ssl context construction", cause=e)
def runner(please_stop):
Log.warning(
"ActiveData listening on encrypted port {{port}}", port=ssl_flask.port
)
flask_app.run(**ssl_flask)
Thread.run("SSL Server", runner)
if flask_config.ssl_context and flask_config.port != 80:
Log.warning(
"ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}",
port=flask_config.port,
)
flask_config.ssl_context = None
def __init__(
self,
name,
slow_queue, # THE SLOWER QUEUE
batch_size=None, # THE MAX SIZE OF BATCHES SENT TO THE SLOW QUEUE
max_size=None, # SET THE MAXIMUM SIZE OF THE QUEUE, WRITERS WILL BLOCK IF QUEUE IS OVER THIS LIMIT
period=None, # MAX TIME (IN SECONDS) BETWEEN FLUSHES TO SLOWER QUEUE
silent=False, # WRITES WILL COMPLAIN IF THEY ARE WAITING TOO LONG
error_target=None # CALL error_target(error, buffer) **buffer IS THE LIST OF OBJECTS ATTEMPTED**
# BE CAREFUL! THE THREAD MAKING THE CALL WILL NOT BE YOUR OWN!
# DEFAULT BEHAVIOUR: THIS WILL KEEP RETRYING WITH WARNINGS
):
if period != None and not isinstance(period, (int, float, long)):
Log.error("Expecting a float for the period")
period = coalesce(period, 1) # SECONDS
batch_size = coalesce(batch_size,
int(max_size / 2) if max_size else None, 900)
max_size = coalesce(max_size, batch_size * 2) # REASONABLE DEFAULT
Queue.__init__(self, name=name, max=max_size, silent=silent)
self.name = name
self.slow_queue = slow_queue
self.thread = Thread.run("threaded queue for " + name, self.worker_bee,
batch_size, period,
error_target) # parent_thread=self)
def enable_profilers(filename):
global FILENAME
global cprofiler_stats
if cprofiler_stats is not None:
return
if filename:
FILENAME = filename
from mo_threads.threads import ALL_LOCK, ALL, Thread
from mo_threads.queues import Queue
cprofiler_stats = Queue("cprofiler stats")
current_thread = Thread.current()
with ALL_LOCK:
threads = list(ALL.values())
for t in threads:
t.cprofiler = CProfiler()
if t is current_thread:
Log.note("starting cprofile for thread {{name}}", name=t.name)
t.cprofiler.__enter__()
else:
Log.note(
"cprofiler not started for thread {{name}} (already running)",
name=t.name)
def __init__(self, flask_app, db, cookie, table="sessions"):
global SINGLTON
if SINGLTON:
Log.error("Can only handle one session manager at a time")
SINGLTON = self
if is_data(db):
self.db = Sqlite(db)
else:
self.db = db
self.table = table
self.cookie = cookie
self.cookie.max_lifetime = parse(self.cookie.max_lifetime)
self.cookie.inactive_lifetime = parse(self.cookie.inactive_lifetime)
if not self.db.about(self.table):
self.setup()
Thread.run("session monitor", self.monitor)
def enable_profilers(filename):
global FILENAME
global cprofiler_stats
if cprofiler_stats is not None:
return
if filename:
FILENAME = filename
from mo_threads.threads import ALL_LOCK, ALL, Thread
from mo_threads.queues import Queue
cprofiler_stats = Queue("cprofiler stats")
current_thread = Thread.current()
with ALL_LOCK:
threads = list(ALL.values())
for t in threads:
t.cprofiler = CProfiler()
if t is current_thread:
Log.note("starting cprofile for thread {{name}}", name=t.name)
t.cprofiler.__enter__()
else:
Log.note("cprofiler not started for thread {{name}} (already running)", name=t.name)
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from mo_logs import Log
from mo_threads.lock import Lock
from mo_threads.multiprocess import Process
from mo_threads.queues import Queue, ThreadedQueue
from mo_threads.signal import Signal
from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD
from mo_threads.till import Till
Log.cprofiler_stats = Queue(
"cprofiler stats") # ACCUMULATION OF STATS FROM ALL THREADS
MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon)
MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers)
# from threading import Thread as _threading_Thread
# _temp = _threading_Thread.setDaemon
#
# fixes = []
# # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR
# try:
# from paramiko import Transport
#
# def fix(self):
# if isinstance(self, Transport):
# self.stop = self.close # WE KNOW Transport DOES NOT HAVE A stop() METHOD, SO ADDING SHOULD BE FINE
# parent = Thread.current()
# parent.add_child(self)
def __init__(
self,
name,
queue, # THE SLOWER QUEUE
batch_size=None, # THE MAX SIZE OF BATCHES SENT TO THE SLOW QUEUE
max_size=None, # SET THE MAXIMUM SIZE OF THE QUEUE, WRITERS WILL BLOCK IF QUEUE IS OVER THIS LIMIT
period=None, # MAX TIME (IN SECONDS) BETWEEN FLUSHES TO SLOWER QUEUE
silent=False, # WRITES WILL COMPLAIN IF THEY ARE WAITING TOO LONG
error_target=None # CALL THIS WITH ERROR **AND THE LIST OF OBJECTS ATTEMPTED**
# BE CAREFUL! THE THREAD MAKING THE CALL WILL NOT BE YOUR OWN!
# DEFAULT BEHAVIOUR: THIS WILL KEEP RETRYING WITH WARNINGS
):
if period != None and not isinstance(period, (int, float, long)):
Log.error("Expecting a float for the period")
batch_size = coalesce(batch_size,
int(max_size / 2) if max_size else None, 900)
max_size = coalesce(max_size, batch_size * 2) # REASONABLE DEFAULT
period = coalesce(period, 1) # SECONDS
Queue.__init__(self, name=name, max=max_size, silent=silent)
def worker_bee(please_stop):
please_stop.on_go(lambda: self.add(THREAD_STOP))
_buffer = []
_post_push_functions = []
now = time()
next_push = Till(till=now + period) # THE TIME WE SHOULD DO A PUSH
last_push = now - period
def push_to_queue():
queue.extend(_buffer)
del _buffer[:]
for ppf in _post_push_functions:
ppf()
del _post_push_functions[:]
while not please_stop:
try:
if not _buffer:
item = self.pop()
now = time()
if now > last_push + period:
# Log.note("delay next push")
next_push = Till(till=now + period)
else:
item = self.pop(till=next_push)
now = time()
if item is THREAD_STOP:
push_to_queue()
please_stop.go()
break
elif isinstance(item, types.FunctionType):
_post_push_functions.append(item)
elif item is not None:
_buffer.append(item)
except Exception as e:
e = Except.wrap(e)
if error_target:
try:
error_target(e, _buffer)
except Exception as f:
Log.warning(
"`error_target` should not throw, just deal",
name=name,
cause=f)
else:
Log.warning("Unexpected problem", name=name, cause=e)
try:
if len(_buffer) >= batch_size or next_push:
if _buffer:
push_to_queue()
last_push = now = time()
next_push = Till(till=now + period)
except Exception as e:
e = Except.wrap(e)
if error_target:
try:
error_target(e, _buffer)
except Exception as f:
Log.warning(
"`error_target` should not throw, just deal",
name=name,
cause=f)
else:
Log.warning(
"Problem with {{name}} pushing {{num}} items to data sink",
name=name,
num=len(_buffer),
cause=e)
if _buffer:
# ONE LAST PUSH, DO NOT HAVE TIME TO DEAL WITH ERRORS
push_to_queue()
self.thread = Thread.run("threaded queue for " + name,
worker_bee) # parent_thread=self)
# THIS THREADING MODULE IS PERMEATED BY THE please_stop SIGNAL.
# THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS
# FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from mo_threads.lock import Lock
from mo_threads.multiprocess import Process
from mo_threads.queues import Queue, ThreadedQueue
from mo_threads.signal import Signal
from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD
from mo_threads.till import Till
MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon)
MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers)
# from threading import Thread as _threading_Thread
# _temp = _threading_Thread.setDaemon
#
# fixes = []
# # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR
# try:
# from paramiko import Transport
#
# def fix(self):
"`error_target` should not throw, just deal",
name=name,
cause=f)
else:
_Log.warning(
"Problem with {{name}} pushing {{num}} items to data sink",
name=name,
num=len(_buffer),
cause=e)
if _buffer:
# ONE LAST PUSH, DO NOT HAVE TIME TO DEAL WITH ERRORS
push_to_queue()
self.thread = Thread.run("threaded queue for " + name,
worker_bee,
parent_thread=self)
def add(self, value, timeout=None):
with self.lock:
self._wait_for_queue_space(timeout=timeout)
if not self.please_stop:
self.queue.append(value)
# if Random.range(0, 50) == 0:
# sizes = wrap([{"id":i["id"], "size":len(convert.value2json(i))} for i in self.queue if isinstance(i, Mapping)])
# size=sum(sizes.size)
# if size>50000000:
# from pyLibrary.queries import jx
#
# biggest = jx.sort(sizes, "size").last().id
# _Log.note("Big record {{id}}", id=biggest)
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
