def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( [str(p) for p in params], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath, env={str(k): str(v) for k, v in set_default(env, os.environ).items()}, shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, target, *args, **kwargs): self.id = -1 self.name = name self.target = target self.end_of_thread = None self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get( "please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) self.cprofiler = Null self.children = [] if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = coalesce(name, "thread_" + text(object.__hash__(self))) self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.please_stop = self.kwargs.get(PLEASE_STOP) if self.please_stop is None: self.please_stop = self.kwargs[PLEASE_STOP] = Signal( "please_stop for " + self.name ) self.thread = None self.join_attempt = Signal("joining with " + self.name) self.stopped = Signal("stopped signal for " + self.name) if PARENT_THREAD in kwargs: del self.kwargs[PARENT_THREAD] self.parent = kwargs[PARENT_THREAD] else: self.parent = Thread.current() self.parent.add_child(self)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): """ Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done via proper thread-safe queues of the same name. Since the process is managed and monitored by threads, the main thread is not blocked when the child process encounters problems :param name: name given to this process :param params: list of strings for program name and parameters :param cwd: current working directory :param env: enviroment variables :param debug: true to be verbose about stdin/stdout :param shell: true to run as command line :param bufsize: if you want to screw stuff up """ self.debug = debug or DEBUG self.process_id = Process.next_process_id Process.next_process_id += 1 self.name = name + " (" + text(self.process_id) + ")" self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug) self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug) self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug) try: if cwd == None: cwd = os.getcwd() else: cwd = str(cwd) command = [str(p) for p in params] self.debug and Log.note("command: {{command}}", command=command) self.service = service = subprocess.Popen( [str(p) for p in params], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd, env={str(k): str(v) for k, v in set_default(env, os.environ).items()}, shell=shell ) self.please_stop = Signal() self.please_stop.then(self._kill) self.child_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ self.name = name self.max = coalesce(max, 2 ** 10) self.silent = silent self.allow_add_after_close=allow_add_after_close self.unique = unique self.please_stop = Signal("stop signal for " + name) self.lock = Lock("lock for queue " + name) self.queue = deque() self.next_warning = time() # FOR DEBUGGING
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ self.name = name self.max = coalesce(max, 2**10) self.silent = silent self.allow_add_after_close = allow_add_after_close self.unique = unique self.closed = Signal( "stop adding signal for " + name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE self.lock = Lock("lock for queue " + name) self.queue = deque()
def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = coalesce(name, "thread_" + text_type(object.__hash__(self))) self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
class Command(object): """ FASTER Process CLASS - OPENS A COMMAND_LINE APP (CMD on windows) AND KEEPS IT OPEN FOR MULTIPLE COMMANDS EACH WORKING DIRECTORY WILL HAVE ITS OWN PROCESS, MULTIPLE PROCESSES WILL OPEN FOR THE SAME DIR IF MULTIPLE THREADS ARE REQUESTING Commands """ available_locker = Lock("cmd lock") available_process = {} def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): shell = True self.name = name self.key = (cwd, wrap(env), debug, shell) self.stdout = Queue("stdout for " + name) self.stderr = Queue("stderr for " + name) with Command.available_locker: avail = Command.available_process.setdefault(self.key, []) if not avail: self.process = Process("command shell", [cmd()], cwd, env, debug, shell, bufsize) self.process.stdin.add(set_prompt()) self.process.stdin.add("echo %errorlevel%") _wait_for_start(self.process.stdout, Null) else: self.process = avail.pop() self.process.stdin.add(" ".join(cmd_escape(p) for p in params)) self.process.stdin.add("echo %errorlevel%") self.stdout_thread = Thread.run("", self._stream_relay, self.process.stdout, self.stdout) self.stderr_thread = Thread.run("", self._stream_relay, self.process.stderr, self.stderr) self.returncode = None def join(self, raise_on_error=False, till=None): try: try: # WAIT FOR COMMAND LINE RESPONSE ON stdout self.stdout_thread.join() except Exception as e: Log.error("unexpected problem processing stdout", cause=e) try: self.stderr_thread.please_stop.go() self.stderr_thread.join() except Exception as e: Log.error("unexpected problem processing stderr", cause=e) if raise_on_error and self.returncode != 0: Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}", process=self.name, code=self.returncode, stderr=list(self.stderr)) return self finally: with Command.available_locker: Command.available_process[self.key].append(self.process) def _stream_relay(self, source, destination, please_stop=None): """ :param source: :param destination: :param error: Throw error if line shows up :param please_stop: :return: """ prompt_count = 0 prompt = PROMPT + ">" line_count = 0 while not please_stop: value = source.pop(till=please_stop) if value is None: destination.add(THREAD_STOP) return elif value is THREAD_STOP: destination.add(THREAD_STOP) return elif line_count == 0 and "is not recognized as an internal or external command" in value: Log.error("Problem with command: {{desc}}", desc=value) elif value.startswith(prompt): if prompt_count: # GET THE ERROR LEVEL self.returncode = int(source.pop(till=please_stop)) destination.add(THREAD_STOP) return else: prompt_count += 1 else: line_count += 1 destination.add(value)
else: cr_count = -1000000 # NOT /dev/null if line.strip() == "exit": Log.alert("'exit' Detected! Stopping...") return def _wait_for_interrupt(please_stop): DEBUG and Log.note("inside wait-for-shutdown loop") while not please_stop: try: sleep(1) except Exception: pass def _interrupt_main_safely(): try: interrupt_main() except KeyboardInterrupt: # WE COULD BE INTERRUPTING SELF pass MAIN_THREAD = MainThread() ALL_LOCK = Lock("threads ALL_LOCK") ALL = dict() ALL[get_ident()] = MAIN_THREAD
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + string2quote(name)) self.stdin = Queue("stdin for process " + string2quote(name), silent=True) self.stdout = Queue("stdout for process " + string2quote(name), silent=True) self.stderr = Queue("stderr for process " + string2quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception, e: Log.error("Can not call", e)