Example #1
0
class PooledThread(Resource):
    def __init__(self, name, release):
        Resource.__init__(self, name)
        self._release = release
        self._ready, self._queue = Event(), InterlockedQueue()
        if __name__ == "__main__":
            self._timeout = Timeout(3.0)
        else:
            self._timeout = Timeout(60.0)
        self._count = 0

    def _expired(self):
        return self._timeout.expired or Resource._expired(self)

    def connect(self):
        Resource.connect(self)
        self._thread = LightThread(target=self._thread_proc,
                                   name="{0:s}:?".format(self.name))
        self._thread.start()
        self._ready.wait(
            3.0)  # this may spend waiting slightly less, but it's ok
        if not self._ready.is_set():
            self._queue.push(
                exit)  # just in case the thread has in fact started
            raise Exception("new thread failed to start in 3.0 seconds")

    def _thread_proc(self):
        self._ready.set()
        while True:  # exits upon processing of exit pushed in disconnect()
            try:
                self._count += 1
                thread_name = "{0:s}:{1:d}".format(self.name, self._count)
                current_thread().name = thread_name
                work_unit = self._queue.pop()
                work_unit()
                self._timeout.reset()
            finally:
                self._release(
                    self)  # this actually invokes ThreadPool._release

    # this method may be called by external thread (ex. pool sweep)
    # or by this thread itself, and posts an exit kind of work unit

    def disconnect(self):
        try:
            if current_thread() is not self._thread:
                self._queue.push(exit)
                self._thread.join(3.0)
        finally:
            Resource.disconnect(self)

    # this method is called by the thread pool to post a work unit
    # to this thread, as well as by the thread itself at disconnect

    def push(self, work_unit):
        self._queue.push(work_unit)
Example #2
0
class PooledThread(Resource):

    def __init__(self, name, release):
        Resource.__init__(self, name)
        self._release = release
        self._ready, self._queue = Event(), InterlockedQueue()
        if __name__ == "__main__":
            self._timeout = Timeout(3.0)
        else:
            self._timeout = Timeout(60.0)
        self._count = 0

    def _expired(self):
        return self._timeout.expired or Resource._expired(self)

    def connect(self):
        Resource.connect(self)
        self._thread = LightThread(target = self._thread_proc,
                                   name = "{0:s}:?".format(self.name))
        self._thread.start()
        self._ready.wait(3.0) # this may spend waiting slightly less, but it's ok
        if not self._ready.is_set():
            self._queue.push(exit) # just in case the thread has in fact started
            raise Exception("new thread failed to start in 3.0 seconds")

    def _thread_proc(self):
        self._ready.set()
        while True: # exits upon processing of exit pushed in disconnect()
            try:
                self._count += 1
                thread_name = "{0:s}:{1:d}".format(self.name, self._count)
                current_thread().name = thread_name
                work_unit = self._queue.pop()
                work_unit()
                self._timeout.reset()
            finally:
                self._release(self) # this actually invokes ThreadPool._release

    # this method may be called by external thread (ex. pool sweep)
    # or by this thread itself, and posts an exit kind of work unit

    def disconnect(self):
        try:
            if current_thread() is not self._thread:
                self._queue.push(exit)
                self._thread.join(3.0)
        finally:
            Resource.disconnect(self)

    # this method is called by the thread pool to post a work unit
    # to this thread, as well as by the thread itself at disconnect

    def push(self, work_unit):
        self._queue.push(work_unit)
Example #3
0
def primary_startup(
        node_cage: by_regex("^[A-Za-z0-9_-]{1,32}(\\.[A-Za-z0-9_-]{1,32})?$")):

    if "." in node_cage:  # node name is specified explicitly
        node, cage = node_cage.split(".")
    else:  # node name is taken from the environment
        cage = node_cage
        node = node_name().split(".")[0]

    cage_dir = os_path.join(cages_dir, cage)  # cage directory must exist
    if not os_path.isdir(cage_dir):
        raise Exception("cage directory does not exist")

    logs_dir = os_path.join(
        cage_dir, "logs")  # while logs directory will be created if necessary
    if not os_path.isdir(logs_dir):
        try:
            mkdir(logs_dir)
        except OSError as e:
            if e.errno != EEXIST:
                raise

    # write own pid file, this also serves as a test of the logs directory writability

    with open(os_path.join(logs_dir, "{0:s}.pid".format(cage)), "wb") as f:
        f.write("{0:d}".format(getpid()).encode("ascii"))

    restarting_after_failure = False  # performing normal startup by default

    while True:  # keep starting secondary startup script until it exits successfully

        def drain_stream(stream):
            try:
                while stream.read(512):
                    pass
            except:
                pass  # just exit

        # pass the same arguments to the same script, only prefixing them with dash

        startup_py = popen(python, os_path.join(pmnc_dir,
                                                "startup.py"), "-", node, cage,
                           restarting_after_failure and "FAILURE" or "NORMAL")

        # any output from the secondary script is ignored

        stdout_reader = LightThread(target=drain_stream,
                                    args=(startup_py.stdout, ))
        stdout_reader.start()

        stderr_reader = LightThread(target=drain_stream,
                                    args=(startup_py.stderr, ))
        stderr_reader.start()

        # wait for the secondary script to terminate

        while startup_py.poll() is None:
            try:
                sleep(
                    3.0
                )  # fails with "interrupted system call" at logoff when started as win32 service
            except:
                pass

        stdout_reader.join(3.0)  # should have exited with eof
        stderr_reader.join(3.0)  # or broken pipe, but who knows

        if startup_py.wait() != 0:
            restarting_after_failure = True  # set the flag and restart the secondary script
        else:
            break  # successful exit