def restart_thrift_clients(self, restart=True): if self.client_sync and restart: self.client_sync.close() self.client_sync = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=False) if self.client_pool and restart: self.client_pool.close() self.client_pool = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=5) if self.client_loop and restart: self.client_loop.close() self.client_loop = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=True, network_timeout=7200000)
from hyphe_backend.memorystructure import MemoryStructure as ms import time from hyphe_backend.lib.thriftpool import ThriftPooledClient, ThriftSyncClient from twisted.internet import reactor, threads cl1 = ThriftSyncClient(iface_cls=ms.Client, host='localhost', port=9090, retries=3, framed_transport=True, compact_protocol=False) cl2 = ThriftPooledClient(iface_cls=ms.Client, host='localhost', port=9090, pool_size=5, retries=3, framed_transport=True, compact_protocol=False) cl3 = ThriftPooledClient(iface_cls=ms.Client, host='localhost', port=9090, pool_size=3, retries=3, framed_transport=True, compact_protocol=False) cl4 = ThriftPooledClient(iface_cls=ms.Client, host='localhost', port=9090, pool_size=3, retries=3,
class LuceneCorpus(Thread): daemon = True def __init__(self, factory, name, ram=256, keepalive=1800, loglevel="INFO", quiet=False): Thread.__init__(self) self.factory = factory self.status = "init" self.name = name self.ram = ram self.port = 0 self.loglevel = loglevel self.quiet = quiet self.command = "" self.error = None self.processus = None self.client_sync = None self.client_pool = None self.client_loop = None self.lastcall = time.time() self.loop_running = False self.keepalive = keepalive self.monitor = LoopingCall(self.__check_timeout__) def restart_thrift_clients(self, restart=True): if self.client_sync and restart: self.client_sync.close() self.client_sync = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=False) if self.client_pool and restart: self.client_pool.close() self.client_pool = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=5) if self.client_loop and restart: self.client_loop.close() self.client_loop = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=True, network_timeout=7200000) def __check_timeout__(self): delay = time.time() - self.lastcall if self.status == "ready" and self.keepalive < delay and not self.loop_running: self.log("Stopping after %ss of inactivity" % int(delay)) self.stop() def log(self, msg, error=False): self.factory.log(self.name, msg, error, quiet=self.quiet) def stopping(self): return self.status in ["stopping", "stopped", "error"] def stop(self): if self.monitor.running: self.monitor.stop() if self.port and self.port not in self.factory.ports_free: self.factory.ports_free.append(self.port) self.factory.ram_free += self.ram self.port = None if self.stopping(): return self.status = "error" if self.error else "stopping" if self.client_loop: self.client_loop.close() if self.client_sync: self.client_sync.close() if self.client_pool: self.client_pool.close() if self.processus and not self.processus.poll(): self.processus.terminate() self.log("MemoryStructure stopped") if not self.error: self.status = "stopped" def hard_restart(self): self.status = "restarting" pscommand = lambda x: ['p%s' % x, '-f', ' corpus=%s ' % self.name] subprocess.call(pscommand("kill")) stoptime = time.time() + 30 with open(os.devnull, "w") as fnull: while not subprocess.call(pscommand("grep"), stdout=fnull): if time.time() > stoptime: self.log("Couldn't stop existing corpus", True) self.stop() return self.factory.start_corpus(self.name) def choose_port(self): ports = list(self.factory.ports_free) shuffle(ports) s = socket.socket() for port in ports: try: s.connect(("", port)) s.close() except socket.error: self.port = port self.factory.ports_free.remove(port) break def run(self): self.port = 0 self.error = None stoptime = time.time() + 5 while not self.port and time.time() < stoptime: self.choose_port() if not self.port: self.log("Couldn't find a port to attach MemoryStructure to", True) self.stop() return self.factory.ram_free -= self.ram if self.factory.ram_free < 0: self.log("Couldn't find enough ram to start MemoryStructure", True) self.stop() return self.status = "started" size = min(128, max(32, int(self.ram / 4))) java_options = "-Xms%dm -Xmx%dm " % (self.ram, self.ram) java_options += "-XX:NewSize=%sm -XX:MaxNewSize=%sm " % (size, size) java_options += "-XX:NewRatio=3 -XX:SurvivorRatio=6 " java_options += "-XX:PermSize=%sm -XX:MaxPermSize=%sm " % (size, size) java_options += "-XX:+UseParallelGC -XX:ParallelGCThreads=2" self.command = "java -server %s -jar %s " % (java_options, HYPHE_MS_JAR) self.command += "corpus=%s thrift.port=%d log.level=%s" % \ (self.name, self.port, self.loglevel) self.log("Starting MemoryStructure on port " + \ "%s with %sMo ram for at least %ss (%sMo ram and %s ports left)" % \ (self.port, self.ram, self.keepalive, self.factory.ram_free, len(self.factory.ports_free))) self.processus = subprocess.Popen(self.command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) self.restart_thrift_clients() while self.processus.poll() is None: line = self.processus.stdout.readline().strip('\n') if not line.strip(): continue try: lts, ltype, lclass, lthread, msg = parse_log(line) except: if line.startswith("java.lang.OutOfMemoryError"): if self.factory.ram_free >= 256: self.log("Java heap space, trying to restart " + \ "with 256Mo more ram", True) self.ram += 256 else: self.log("Not enough ram to increase corpus size, " + \ "trying to restart anyway", True) self.hard_restart() break # skip tracebacks elif line.startswith('\tat ') or line.startswith( 'java.') or indexClosed(line): continue self.log(line, True) self.stop() else: if lclass=="LRUIndex" and \ msg.startswith("starting Thrift server"): self.status = "ready" self.lastcall = time.time() self.monitor.start(max(1, int(self.keepalive / 6))) self.log("MemoryStructure ready") elif ltype == "ERROR": if msg.startswith("Lock obtain timed out") or \ "Could not create ServerSocket" in msg: self.log("WARNING: Corpus seems already running," + \ "trying to stop and restart it...") self.hard_restart() break # skip these errors as they are followed by a more explicit stacktrace elif "Unexpected throwable while invoking!" in msg or indexClosed( msg): pass else: self.log(msg) elif msg == "closed": if not self.stopping(): self.status = "stopping" self.log(msg) if not self.stopping(): self.log("MemoryStructure crashed", True) self.stop()
class LuceneCorpus(Thread): daemon = True def __init__(self, factory, name, ram=256, keepalive=1800, loglevel="INFO", quiet=False): Thread.__init__(self) self.factory = factory self.status = "init" self.name = name self.ram = ram self.port = 0 self.loglevel = loglevel self.quiet = quiet self.command = "" self.error = None self.processus = None self.client_sync = None self.client_pool = None self.client_loop = None self.lastcall = time.time() self.loop_running = False self.keepalive = keepalive self.monitor = LoopingCall(self.__check_timeout__) def restart_thrift_clients(self, restart=True): if self.client_sync and restart: self.client_sync.close() self.client_sync = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=False) if self.client_pool and restart: self.client_pool.close() self.client_pool = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=5) if self.client_loop and restart: self.client_loop.close() self.client_loop = ThriftPooledClient(ms.Client, host="localhost", port=self.port, pool_size=1, async=True, network_timeout=7200000) def __check_timeout__(self): delay = time.time() - self.lastcall if self.status == "ready" and self.keepalive < delay and not self.loop_running: self.log("Stopping after %ss of inactivity" % int(delay)) self.stop() def log(self, msg, error=False): self.factory.log(self.name, msg, error, quiet=self.quiet) def stopping(self): return self.status in ["stopping", "stopped", "error"] def stop(self): if self.monitor.running: self.monitor.stop() if self.port and self.port not in self.factory.ports_free: self.factory.ports_free.append(self.port) self.factory.ram_free += self.ram self.port = None if self.stopping(): return self.status = "error" if self.error else "stopping" if self.client_loop: self.client_loop.close() if self.client_sync: self.client_sync.close() if self.client_pool: self.client_pool.close() if self.processus and not self.processus.poll(): self.processus.terminate() self.log("MemoryStructure stopped") if not self.error: self.status = "stopped" def hard_restart(self): self.status = "restarting" pscommand = lambda x: ['p%s' % x, '-f', ' corpus=%s ' % self.name] subprocess.call(pscommand("kill")) stoptime = time.time() + 30 with open(os.devnull, "w") as fnull: while not subprocess.call(pscommand("grep"), stdout=fnull): if time.time() > stoptime: self.log("Couldn't stop existing corpus", True) self.stop() return self.factory.start_corpus(self.name) def choose_port(self): ports = list(self.factory.ports_free) shuffle(ports) s = socket.socket() for port in ports: try: s.connect(("", port)) s.close() except socket.error: self.port = port self.factory.ports_free.remove(port) break def run(self): self.port = 0 self.error = None stoptime = time.time() + 5 while not self.port and time.time() < stoptime: self.choose_port() if not self.port: self.log("Couldn't find a port to attach MemoryStructure to", True) self.stop() return self.factory.ram_free -= self.ram if self.factory.ram_free < 0: self.log("Couldn't find enough ram to start MemoryStructure", True) self.stop() return self.status = "started" size = min(128, max(32, int(self.ram/4))) java_options = "-Xms%dm -Xmx%dm " % (self.ram, self.ram) java_options += "-XX:NewSize=%sm -XX:MaxNewSize=%sm " % (size, size) java_options += "-XX:NewRatio=3 -XX:SurvivorRatio=6 " java_options += "-XX:PermSize=%sm -XX:MaxPermSize=%sm " % (size, size) java_options += "-XX:+UseParallelGC -XX:ParallelGCThreads=2" self.command = "java -server %s -jar %s " % (java_options, HYPHE_MS_JAR) self.command += "corpus=%s thrift.port=%d log.level=%s" % \ (self.name, self.port, self.loglevel) self.log("Starting MemoryStructure on port " + \ "%s with %sMo ram for at least %ss (%sMo ram and %s ports left)" % \ (self.port, self.ram, self.keepalive, self.factory.ram_free, len(self.factory.ports_free))) self.processus = subprocess.Popen(self.command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) self.restart_thrift_clients() while self.processus.poll() is None: line = self.processus.stdout.readline().strip('\n') if not line.strip(): continue try: lts, ltype, lclass, lthread, msg = parse_log(line) except: if line.startswith("java.lang.OutOfMemoryError"): if self.factory.ram_free >= 256: self.log("Java heap space, trying to restart " + \ "with 256Mo more ram", True) self.ram += 256 else: self.log("Not enough ram to increase corpus size, " + \ "trying to restart anyway", True) self.hard_restart() break # skip tracebacks elif line.startswith('\tat ') or line.startswith('java.') or indexClosed(line): continue self.log(line, True) self.stop() else: if lclass=="LRUIndex" and \ msg.startswith("starting Thrift server"): self.status = "ready" self.lastcall = time.time() self.monitor.start(max(1,int(self.keepalive/6))) self.log("MemoryStructure ready") elif ltype == "ERROR": if msg.startswith("Lock obtain timed out") or \ "Could not create ServerSocket" in msg: self.log("WARNING: Corpus seems already running," + \ "trying to stop and restart it...") self.hard_restart() break # skip these errors as they are followed by a more explicit stacktrace elif "Unexpected throwable while invoking!" in msg or indexClosed(msg): pass else: self.log(msg) elif msg == "closed": if not self.stopping(): self.status = "stopping" self.log(msg) if not self.stopping(): self.log("MemoryStructure crashed", True) self.stop()