def restart_thrift_clients(self, restart=True):
if self.client_sync and restart:
self.client_sync.close()
self.client_sync = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=False)
if self.client_pool and restart:
self.client_pool.close()
self.client_pool = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=5)
if self.client_loop and restart:
self.client_loop.close()
self.client_loop = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=True, network_timeout=7200000)
def restart_thrift_clients(self, restart=True):
if self.client_sync and restart:
self.client_sync.close()
self.client_sync = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=False)
if self.client_pool and restart:
self.client_pool.close()
self.client_pool = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=5)
if self.client_loop and restart:
self.client_loop.close()
self.client_loop = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=True, network_timeout=7200000)
from hyphe_backend.memorystructure import MemoryStructure as ms
import time
from hyphe_backend.lib.thriftpool import ThriftPooledClient, ThriftSyncClient
from twisted.internet import reactor, threads
cl1 = ThriftSyncClient(iface_cls=ms.Client,
host='localhost',
port=9090,
retries=3,
framed_transport=True,
compact_protocol=False)
cl2 = ThriftPooledClient(iface_cls=ms.Client,
host='localhost',
port=9090,
pool_size=5,
retries=3,
framed_transport=True,
compact_protocol=False)
cl3 = ThriftPooledClient(iface_cls=ms.Client,
host='localhost',
port=9090,
pool_size=3,
retries=3,
framed_transport=True,
compact_protocol=False)
cl4 = ThriftPooledClient(iface_cls=ms.Client,
host='localhost',
port=9090,
pool_size=3,
retries=3,
class LuceneCorpus(Thread):
daemon = True
def __init__(self,
factory,
name,
ram=256,
keepalive=1800,
loglevel="INFO",
quiet=False):
Thread.__init__(self)
self.factory = factory
self.status = "init"
self.name = name
self.ram = ram
self.port = 0
self.loglevel = loglevel
self.quiet = quiet
self.command = ""
self.error = None
self.processus = None
self.client_sync = None
self.client_pool = None
self.client_loop = None
self.lastcall = time.time()
self.loop_running = False
self.keepalive = keepalive
self.monitor = LoopingCall(self.__check_timeout__)
def restart_thrift_clients(self, restart=True):
if self.client_sync and restart:
self.client_sync.close()
self.client_sync = ThriftPooledClient(ms.Client,
host="localhost",
port=self.port,
pool_size=1,
async=False)
if self.client_pool and restart:
self.client_pool.close()
self.client_pool = ThriftPooledClient(ms.Client,
host="localhost",
port=self.port,
pool_size=5)
if self.client_loop and restart:
self.client_loop.close()
self.client_loop = ThriftPooledClient(ms.Client,
host="localhost",
port=self.port,
pool_size=1,
async=True,
network_timeout=7200000)
def __check_timeout__(self):
delay = time.time() - self.lastcall
if self.status == "ready" and self.keepalive < delay and not self.loop_running:
self.log("Stopping after %ss of inactivity" % int(delay))
self.stop()
def log(self, msg, error=False):
self.factory.log(self.name, msg, error, quiet=self.quiet)
def stopping(self):
return self.status in ["stopping", "stopped", "error"]
def stop(self):
if self.monitor.running:
self.monitor.stop()
if self.port and self.port not in self.factory.ports_free:
self.factory.ports_free.append(self.port)
self.factory.ram_free += self.ram
self.port = None
if self.stopping():
return
self.status = "error" if self.error else "stopping"
if self.client_loop:
self.client_loop.close()
if self.client_sync:
self.client_sync.close()
if self.client_pool:
self.client_pool.close()
if self.processus and not self.processus.poll():
self.processus.terminate()
self.log("MemoryStructure stopped")
if not self.error:
self.status = "stopped"
def hard_restart(self):
self.status = "restarting"
pscommand = lambda x: ['p%s' % x, '-f', ' corpus=%s ' % self.name]
subprocess.call(pscommand("kill"))
stoptime = time.time() + 30
with open(os.devnull, "w") as fnull:
while not subprocess.call(pscommand("grep"), stdout=fnull):
if time.time() > stoptime:
self.log("Couldn't stop existing corpus", True)
self.stop()
return
self.factory.start_corpus(self.name)
def choose_port(self):
ports = list(self.factory.ports_free)
shuffle(ports)
s = socket.socket()
for port in ports:
try:
s.connect(("", port))
s.close()
except socket.error:
self.port = port
self.factory.ports_free.remove(port)
break
def run(self):
self.port = 0
self.error = None
stoptime = time.time() + 5
while not self.port and time.time() < stoptime:
self.choose_port()
if not self.port:
self.log("Couldn't find a port to attach MemoryStructure to", True)
self.stop()
return
self.factory.ram_free -= self.ram
if self.factory.ram_free < 0:
self.log("Couldn't find enough ram to start MemoryStructure", True)
self.stop()
return
self.status = "started"
size = min(128, max(32, int(self.ram / 4)))
java_options = "-Xms%dm -Xmx%dm " % (self.ram, self.ram)
java_options += "-XX:NewSize=%sm -XX:MaxNewSize=%sm " % (size, size)
java_options += "-XX:NewRatio=3 -XX:SurvivorRatio=6 "
java_options += "-XX:PermSize=%sm -XX:MaxPermSize=%sm " % (size, size)
java_options += "-XX:+UseParallelGC -XX:ParallelGCThreads=2"
self.command = "java -server %s -jar %s " % (java_options,
HYPHE_MS_JAR)
self.command += "corpus=%s thrift.port=%d log.level=%s" % \
(self.name, self.port, self.loglevel)
self.log("Starting MemoryStructure on port " + \
"%s with %sMo ram for at least %ss (%sMo ram and %s ports left)" % \
(self.port, self.ram, self.keepalive,
self.factory.ram_free, len(self.factory.ports_free)))
self.processus = subprocess.Popen(self.command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True)
self.restart_thrift_clients()
while self.processus.poll() is None:
line = self.processus.stdout.readline().strip('\n')
if not line.strip():
continue
try:
lts, ltype, lclass, lthread, msg = parse_log(line)
except:
if line.startswith("java.lang.OutOfMemoryError"):
if self.factory.ram_free >= 256:
self.log("Java heap space, trying to restart " + \
"with 256Mo more ram", True)
self.ram += 256
else:
self.log("Not enough ram to increase corpus size, " + \
"trying to restart anyway", True)
self.hard_restart()
break
# skip tracebacks
elif line.startswith('\tat ') or line.startswith(
'java.') or indexClosed(line):
continue
self.log(line, True)
self.stop()
else:
if lclass=="LRUIndex" and \
msg.startswith("starting Thrift server"):
self.status = "ready"
self.lastcall = time.time()
self.monitor.start(max(1, int(self.keepalive / 6)))
self.log("MemoryStructure ready")
elif ltype == "ERROR":
if msg.startswith("Lock obtain timed out") or \
"Could not create ServerSocket" in msg:
self.log("WARNING: Corpus seems already running," + \
"trying to stop and restart it...")
self.hard_restart()
break
# skip these errors as they are followed by a more explicit stacktrace
elif "Unexpected throwable while invoking!" in msg or indexClosed(
msg):
pass
else:
self.log(msg)
elif msg == "closed":
if not self.stopping():
self.status = "stopping"
self.log(msg)
if not self.stopping():
self.log("MemoryStructure crashed", True)
self.stop()
class LuceneCorpus(Thread):
daemon = True
def __init__(self, factory, name, ram=256, keepalive=1800, loglevel="INFO", quiet=False):
Thread.__init__(self)
self.factory = factory
self.status = "init"
self.name = name
self.ram = ram
self.port = 0
self.loglevel = loglevel
self.quiet = quiet
self.command = ""
self.error = None
self.processus = None
self.client_sync = None
self.client_pool = None
self.client_loop = None
self.lastcall = time.time()
self.loop_running = False
self.keepalive = keepalive
self.monitor = LoopingCall(self.__check_timeout__)
def restart_thrift_clients(self, restart=True):
if self.client_sync and restart:
self.client_sync.close()
self.client_sync = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=False)
if self.client_pool and restart:
self.client_pool.close()
self.client_pool = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=5)
if self.client_loop and restart:
self.client_loop.close()
self.client_loop = ThriftPooledClient(ms.Client, host="localhost",
port=self.port, pool_size=1, async=True, network_timeout=7200000)
def __check_timeout__(self):
delay = time.time() - self.lastcall
if self.status == "ready" and self.keepalive < delay and not self.loop_running:
self.log("Stopping after %ss of inactivity" % int(delay))
self.stop()
def log(self, msg, error=False):
self.factory.log(self.name, msg, error, quiet=self.quiet)
def stopping(self):
return self.status in ["stopping", "stopped", "error"]
def stop(self):
if self.monitor.running:
self.monitor.stop()
if self.port and self.port not in self.factory.ports_free:
self.factory.ports_free.append(self.port)
self.factory.ram_free += self.ram
self.port = None
if self.stopping():
return
self.status = "error" if self.error else "stopping"
if self.client_loop:
self.client_loop.close()
if self.client_sync:
self.client_sync.close()
if self.client_pool:
self.client_pool.close()
if self.processus and not self.processus.poll():
self.processus.terminate()
self.log("MemoryStructure stopped")
if not self.error:
self.status = "stopped"
def hard_restart(self):
self.status = "restarting"
pscommand = lambda x: ['p%s' % x, '-f', ' corpus=%s ' % self.name]
subprocess.call(pscommand("kill"))
stoptime = time.time() + 30
with open(os.devnull, "w") as fnull:
while not subprocess.call(pscommand("grep"), stdout=fnull):
if time.time() > stoptime:
self.log("Couldn't stop existing corpus", True)
self.stop()
return
self.factory.start_corpus(self.name)
def choose_port(self):
ports = list(self.factory.ports_free)
shuffle(ports)
s = socket.socket()
for port in ports:
try:
s.connect(("", port))
s.close()
except socket.error:
self.port = port
self.factory.ports_free.remove(port)
break
def run(self):
self.port = 0
self.error = None
stoptime = time.time() + 5
while not self.port and time.time() < stoptime:
self.choose_port()
if not self.port:
self.log("Couldn't find a port to attach MemoryStructure to", True)
self.stop()
return
self.factory.ram_free -= self.ram
if self.factory.ram_free < 0:
self.log("Couldn't find enough ram to start MemoryStructure", True)
self.stop()
return
self.status = "started"
size = min(128, max(32, int(self.ram/4)))
java_options = "-Xms%dm -Xmx%dm " % (self.ram, self.ram)
java_options += "-XX:NewSize=%sm -XX:MaxNewSize=%sm " % (size, size)
java_options += "-XX:NewRatio=3 -XX:SurvivorRatio=6 "
java_options += "-XX:PermSize=%sm -XX:MaxPermSize=%sm " % (size, size)
java_options += "-XX:+UseParallelGC -XX:ParallelGCThreads=2"
self.command = "java -server %s -jar %s " % (java_options, HYPHE_MS_JAR)
self.command += "corpus=%s thrift.port=%d log.level=%s" % \
(self.name, self.port, self.loglevel)
self.log("Starting MemoryStructure on port " + \
"%s with %sMo ram for at least %ss (%sMo ram and %s ports left)" % \
(self.port, self.ram, self.keepalive,
self.factory.ram_free, len(self.factory.ports_free)))
self.processus = subprocess.Popen(self.command.split(), stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True)
self.restart_thrift_clients()
while self.processus.poll() is None:
line = self.processus.stdout.readline().strip('\n')
if not line.strip():
continue
try:
lts, ltype, lclass, lthread, msg = parse_log(line)
except:
if line.startswith("java.lang.OutOfMemoryError"):
if self.factory.ram_free >= 256:
self.log("Java heap space, trying to restart " + \
"with 256Mo more ram", True)
self.ram += 256
else:
self.log("Not enough ram to increase corpus size, " + \
"trying to restart anyway", True)
self.hard_restart()
break
# skip tracebacks
elif line.startswith('\tat ') or line.startswith('java.') or indexClosed(line):
continue
self.log(line, True)
self.stop()
else:
if lclass=="LRUIndex" and \
msg.startswith("starting Thrift server"):
self.status = "ready"
self.lastcall = time.time()
self.monitor.start(max(1,int(self.keepalive/6)))
self.log("MemoryStructure ready")
elif ltype == "ERROR":
if msg.startswith("Lock obtain timed out") or \
"Could not create ServerSocket" in msg:
self.log("WARNING: Corpus seems already running," + \
"trying to stop and restart it...")
self.hard_restart()
break
# skip these errors as they are followed by a more explicit stacktrace
elif "Unexpected throwable while invoking!" in msg or indexClosed(msg):
pass
else:
self.log(msg)
elif msg == "closed":
if not self.stopping():
self.status = "stopping"
self.log(msg)
if not self.stopping():
self.log("MemoryStructure crashed", True)
self.stop()
