class TarantoolBoxServer(TarantoolServer): def __new__(cls, core="tarantool", module="box"): return TarantoolServer.__new__(cls) def __init__(self, core="tarantool", module="box"): TarantoolServer.__init__(self, core, module) def configure(self, config): TarantoolServer.configure(self, config) with open(self.config) as fp: dummy_section_name = "tarantool" config = ConfigParser.ConfigParser() config.readfp(TarantoolConfigFile(fp, dummy_section_name)) self.primary_port = self.get_option_int(config, dummy_section_name, "primary_port") self.admin_port = self.get_option_int(config, dummy_section_name, "admin_port") self.memcached_port = self.get_option_int(config, dummy_section_name, "memcached_port") self.port = self.admin_port self.admin = AdminConnection("localhost", self.admin_port) self.sql = BoxConnection("localhost", self.primary_port) if self.memcached_port != 0: # Run memcached client self.memcached = MemcachedConnection('localhost', self.memcached_port) def get_option_int(self, config, section, option): if config.has_option(section, option): return config.getint(section, option) else: return 0 def init(self): # init storage subprocess.check_call( [self.binary, "--init-storage"], cwd=self.vardir, # catch stdout/stderr to not clutter output stdout=subprocess.PIPE, stderr=subprocess.PIPE) def get_param(self, param): data = self.admin.execute("show info", silent=True) info = yaml.load(data)["info"] return info[param] def wait_lsn(self, lsn): while True: curr_lsn = int(self.get_param("lsn")) if (curr_lsn >= lsn): break time.sleep(0.01)
class TarantoolBoxServer(TarantoolServer): def __new__(cls, core="tarantool", module="box"): return TarantoolServer.__new__(cls) def __init__(self, core="tarantool", module="box"): TarantoolServer.__init__(self, core, module) def configure(self, config): TarantoolServer.configure(self, config) with open(self.config) as fp: dummy_section_name = "tarantool" config = ConfigParser.ConfigParser() config.readfp(TarantoolConfigFile(fp, dummy_section_name)) self.primary_port = self.get_option_int(config, dummy_section_name, "primary_port") self.admin_port = self.get_option_int(config, dummy_section_name, "admin_port") self.memcached_port = self.get_option_int(config, dummy_section_name, "memcached_port") self.port = self.admin_port self.admin = AdminConnection("localhost", self.admin_port) self.sql = BoxConnection("localhost", self.primary_port) if self.memcached_port != 0: # Run memcached client self.memcached = MemcachedConnection('localhost', self.memcached_port) def get_option_int(self, config, section, option): if config.has_option(section, option): return config.getint(section, option) else: return 0 def init(self): # init storage subprocess.check_call([self.binary, "--init-storage"], cwd = self.vardir, # catch stdout/stderr to not clutter output stdout = subprocess.PIPE, stderr = subprocess.PIPE) def get_param(self, param): data = self.admin.execute("show info", silent = True) info = yaml.load(data)["info"] return info[param] def wait_lsn(self, lsn): while True: curr_lsn = int(self.get_param("lsn")) if (curr_lsn >= lsn): break time.sleep(0.01)
class TarantoolServer(Server): def __new__(cls, core="tarantool"): return super(Server, cls).__new__(cls) def __init__(self, core="tarantool"): Server.__init__(self, core) self.default_bin_name = "tarantool_box" self.default_config_name = "tarantool.cfg" self.default_init_lua_name = "init.lua" # append additional cleanup patterns self.re_vardir_cleanup += ['*.snap', '*.xlog', '*.inprogress', '*.cfg', '*.sup', '*.lua'] self.process = None self.config = None self.vardir = None self.valgrind_log = "valgrind.log" self.valgrind_sup = os.path.join("share/", "%s.sup" % ('tarantool')) self.init_lua = None self.default_suppression_name = "valgrind.sup" self.pidfile = None self.port = None self.binary = None self.is_started = False self.mem = False self.start_and_exit = False self.gdb = False self.valgrind = False def find_exe(self, builddir, silent=True): "Locate server executable in the build dir or in the PATH." self.builddir = builddir builddir = os.path.join(builddir, "src/box") path = builddir + os.pathsep + os.environ["PATH"] if not silent: print "Looking for server binary in {0} ...".format(path) for dir in path.split(os.pathsep): exe = os.path.join(dir, self.default_bin_name) if os.access(exe, os.X_OK): return exe raise RuntimeError("Can't find server executable in " + path) def install(self, binary=None, vardir=None, mem=None, silent=True): """Install server instance: create necessary directories and files. The server working directory is taken from 'vardir', specified in the program options.""" if vardir != None: self.vardir = vardir if binary != None: self.binary = os.path.abspath(binary) if mem != None: self.mem = mem self.pidfile = os.path.abspath(os.path.join(self.vardir, self.pidfile)) self.valgrind_log = os.path.abspath(os.path.join(self.vardir, self.valgrind_log)) if not silent: print "Installing the server..." print " Found executable at " + self.binary print " Creating and populating working directory in " + self.vardir + "..." if os.access(self.vardir, os.F_OK): if not silent: print " Found old vardir, deleting..." self.kill_old_server() self.cleanup() else: if (self.mem == True and check_tmpfs_exists() and os.path.basename(self.vardir) == self.vardir): create_tmpfs_vardir(self.vardir) else: os.makedirs(self.vardir) shutil.copy(self.config, os.path.join(self.vardir, self.default_config_name)) shutil.copy(self.valgrind_sup, os.path.join(self.vardir, self.default_suppression_name)) var_init_lua = os.path.join(self.vardir, self.default_init_lua_name) if self.init_lua != None: if os.path.exists(var_init_lua): os.remove(var_init_lua) shutil.copy(self.init_lua, var_init_lua) def configure(self, config): def get_option(config, section, key): if not config.has_option(section, key): return None value = config.get(section, key) if value.isdigit(): value = int(value) return value self.config = os.path.abspath(config) # now read the server config, we need some properties from it with open(self.config) as fp: dummy_section_name = "tarantool" config = ConfigParser.ConfigParser() config.readfp(TarantoolConfigFile(fp, dummy_section_name)) self.pidfile = get_option(config, dummy_section_name, "pid_file") self.primary_port = get_option(config, dummy_section_name, "primary_port") self.admin_port = get_option(config, dummy_section_name, "admin_port") self.memcached_port = get_option(config, dummy_section_name, "memcached_port") self.port = self.admin_port self.admin = AdminConnection("localhost", self.admin_port) self.sql = BoxConnection("localhost", self.primary_port) if self.memcached_port != 0: # Run memcached client self.memcached = MemcachedConnection('localhost', self.memcached_port) def reconfigure(self, config, silent=False): if config == None: os.unlink(os.path.join(self.vardir, self.default_config_name)) else: self.config = os.path.abspath(config) shutil.copy(self.config, os.path.join(self.vardir, self.default_config_name)) self.admin.execute("reload configuration", silent=silent) def init(self): # init storage cmd = [self.binary, "--init-storage"] _init = subprocess.Popen(cmd, cwd=self.vardir, stderr = subprocess.STDOUT, stdout = subprocess.PIPE) retcode = _init.wait() if retcode: sys.stderr.write("tarantool_box --init-storage error: \n%s\n" % _init.stdout.read()) raise subprocess.CalledProcessError(retcode, cmd) def get_param(self, param): if param: data = yaml.load(self.admin("box.info." + param, silent=True))[0] else: data = yaml.load(self.admin("show info", silent=True))[info] return data def wait_lsn(self, lsn): while True: curr_lsn = int(self.get_param("lsn")) if (curr_lsn >= lsn): break time.sleep(0.01) def version(self): p = subprocess.Popen([self.binary, "--version"], cwd = self.vardir, stdout = subprocess.PIPE) version = p.stdout.read().rstrip() p.wait() return version def _start_and_exit(self, args, gdb=None, valgrind=None): if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind if self.valgrind: with daemon.DaemonContext(working_directory = self.vardir): subprocess.check_call(args) else: if not self.gdb: args.append("--background") else: raise RuntimeError("'--gdb' and '--start-and-exit' can't be defined together") self.server = subprocess.Popen(args, cwd = self.vardir) self.server.wait() def start(self, start_and_exit=None, gdb=None, valgrind=None, silent=True): if start_and_exit != None: self.start_and_exit = start_and_exit if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind self.debug = self.test_debug() if self.is_started: if not silent: print "The server is already started." return if not silent: print "Starting the server..." version = self.version() print "Starting {0} {1}.".format(os.path.basename(self.binary), version) check_port(self.port) args = self.prepare_args() if self.gdb: args = prepare_gdb(self.binary, args) print "You started the server in gdb mode." print "To attach, use `screen -r tnt-gdb`" elif self.valgrind: args = prepare_valgrind([self.binary] + args, self.valgrind_log, os.path.abspath(os.path.join(self.vardir, self.default_suppression_name))) else: args = [self.binary] + args if self.start_and_exit: self._start_and_exit(args) return self.process = subprocess.Popen(args, cwd = self.vardir) # wait until the server is connected self.wait_until_started() # Set is_started flag, to nicely support cleanup during an exception. self.is_started = True def stop(self, silent=True): """Stop server instance. Do nothing if the server is not started, to properly shut down the server in case of an exception during start up.""" if not self.is_started: if not silent: print "The server is not started." return if not silent: print "Stopping the server..." if self.process == None: self.kill_old_server() return # kill process pid = self.read_pidfile(); if pid != -1: os.kill(pid, signal.SIGTERM) #self.process.kill(signal.SIGTERM) if self.gdb or self.valgrind: time = 0 while time < (1<<30) : if self.process.poll() != None: break time += 1 sleep(1) else: self.process.wait() self.wait_until_stopped() # clean-up processs flags self.is_started = False self.process = None def deploy(self, config=None, binary=None, vardir=None, mem=None, start_and_exit=None, gdb=None, valgrind=None, valgrind_sup=None, init_lua=None, silent=True, need_init=True): if config != None: self.config = config if binary != None: self.binary = binary if vardir != None: self.vardir = vardir if mem != None: self.mem = mem if start_and_exit != None: self.start_and_exit = start_and_exit if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind if init_lua != None: self.init_lua = os.path.abspath(init_lua) else: self.init_lua = None; self.configure(self.config) self.install(self.binary, self.vardir, self.mem, silent) if need_init: self.init() self.start(self.start_and_exit, self.gdb, self.valgrind, silent) def restart(self): self.stop(silent=True) self.start(silent=True) def test_option_get(self, show, option_list_str): args = [self.binary] + option_list_str.split() if show: print " ".join([os.path.basename(self.binary)] + args[1:]) output = subprocess.Popen(args, cwd = self.vardir, stdout = subprocess.PIPE, stderr = subprocess.STDOUT).stdout.read() return output def test_option(self, option_list_str): print self.test_option_get(True, option_list_str) def test_debug(self): output = self.test_option_get(False, "-V") if re.search("-Debug", output): return True return False def kill_old_server(self, silent=True): """Kill old server instance if it exists.""" pid = self.read_pidfile() if pid == -1: return # Nothing to do if not silent: print " Found old server, pid {0}, killing...".format(pid) try: os.kill(pid, signal.SIGTERM) while os.kill(pid, 0) != -1: time.sleep(0.001) except OSError: pass def read_pidfile(self): if os.access(self.pidfile, os.F_OK) == False: # file is inaccessible (not exist or permission denied) return -1 pid = -1 try: with open(self.pidfile) as f: pid = int(f.read()) except: pass return pid def wait_until_started(self): """Wait until the server is started and accepting connections""" while self.read_pidfile() == -1: time.sleep(0.001) is_connected = False while not is_connected: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(("localhost", self.port)) is_connected = True sock.close() except socket.error as e: time.sleep(0.001) def wait_until_stopped(self): """Wait until the server is stoped and has closed sockets""" while self.read_pidfile() != -1: time.sleep(0.001) is_connected = False while not is_connected: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(("localhost", self.port)) is_connected = True sock.close() time.sleep(0.001) continue except socket.error as e: break def find_tests(self, test_suite, suite_path): def patterned(test, patterns): for i in patterns: if test.name.find(i) != -1: return True return False tests = [PythonTest(k, test_suite.args, test_suite.ini) for k in sorted(glob.glob(os.path.join(suite_path, "*.test.py" )))] tests += [LuaTest(k, test_suite.args, test_suite.ini) for k in sorted(glob.glob(os.path.join(suite_path, "*.test.lua")))] test_suite.tests = filter((lambda x: patterned(x, test_suite.args.tests)), tests)
class TarantoolServer(Server): def __new__(cls, core="tarantool"): return super(Server, cls).__new__(cls) def __init__(self, core="tarantool"): Server.__init__(self, core) self.default_bin_name = "tarantool_box" self.default_config_name = "tarantool.cfg" self.default_init_lua_name = "init.lua" # append additional cleanup patterns self.re_vardir_cleanup += [ '*.snap', '*.xlog', '*.inprogress', '*.cfg', '*.sup', '*.lua' ] self.process = None self.config = None self.vardir = None self.valgrind_log = "valgrind.log" self.valgrind_sup = os.path.join("share/", "%s.sup" % ('tarantool')) self.init_lua = None self.default_suppression_name = "valgrind.sup" self.pidfile = None self.port = None self.binary = None self.is_started = False self.mem = False self.start_and_exit = False self.gdb = False self.valgrind = False def find_exe(self, builddir, silent=True): "Locate server executable in the build dir or in the PATH." self.builddir = builddir builddir = os.path.join(builddir, "src/box") path = builddir + os.pathsep + os.environ["PATH"] if not silent: print "Looking for server binary in {0} ...".format(path) for dir in path.split(os.pathsep): exe = os.path.join(dir, self.default_bin_name) if os.access(exe, os.X_OK): return exe raise RuntimeError("Can't find server executable in " + path) def install(self, binary=None, vardir=None, mem=None, silent=True): """Install server instance: create necessary directories and files. The server working directory is taken from 'vardir', specified in the program options.""" if vardir != None: self.vardir = vardir if binary != None: self.binary = os.path.abspath(binary) if mem != None: self.mem = mem self.pidfile = os.path.abspath(os.path.join(self.vardir, self.pidfile)) self.valgrind_log = os.path.abspath( os.path.join(self.vardir, self.valgrind_log)) if not silent: print "Installing the server..." print " Found executable at " + self.binary print " Creating and populating working directory in " + self.vardir + "..." if os.access(self.vardir, os.F_OK): if not silent: print " Found old vardir, deleting..." self.kill_old_server() self.cleanup() else: if (self.mem == True and check_tmpfs_exists() and os.path.basename(self.vardir) == self.vardir): create_tmpfs_vardir(self.vardir) else: os.makedirs(self.vardir) shutil.copy(self.config, os.path.join(self.vardir, self.default_config_name)) shutil.copy(self.valgrind_sup, os.path.join(self.vardir, self.default_suppression_name)) var_init_lua = os.path.join(self.vardir, self.default_init_lua_name) if self.init_lua != None: if os.path.exists(var_init_lua): os.remove(var_init_lua) shutil.copy(self.init_lua, var_init_lua) def configure(self, config): def get_option(config, section, key): if not config.has_option(section, key): return None value = config.get(section, key) if value.isdigit(): value = int(value) return value self.config = os.path.abspath(config) # now read the server config, we need some properties from it with open(self.config) as fp: dummy_section_name = "tarantool" config = ConfigParser.ConfigParser() config.readfp(TarantoolConfigFile(fp, dummy_section_name)) self.pidfile = get_option(config, dummy_section_name, "pid_file") self.primary_port = get_option(config, dummy_section_name, "primary_port") self.admin_port = get_option(config, dummy_section_name, "admin_port") self.memcached_port = get_option(config, dummy_section_name, "memcached_port") self.port = self.admin_port self.admin = AdminConnection("localhost", self.admin_port) self.sql = BoxConnection("localhost", self.primary_port) if self.memcached_port != 0: # Run memcached client self.memcached = MemcachedConnection('localhost', self.memcached_port) def reconfigure(self, config, silent=False): if config == None: os.unlink(os.path.join(self.vardir, self.default_config_name)) else: self.config = os.path.abspath(config) shutil.copy(self.config, os.path.join(self.vardir, self.default_config_name)) self.admin.execute("reload configuration", silent=silent) def init(self): # init storage cmd = [self.binary, "--init-storage"] _init = subprocess.Popen(cmd, cwd=self.vardir, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) retcode = _init.wait() if retcode: sys.stderr.write("tarantool_box --init-storage error: \n%s\n" % _init.stdout.read()) raise subprocess.CalledProcessError(retcode, cmd) def get_param(self, param): if param: data = yaml.load(self.admin("box.info." + param, silent=True))[0] else: data = yaml.load(self.admin("show info", silent=True))[info] return data def wait_lsn(self, lsn): while True: curr_lsn = int(self.get_param("lsn")) if (curr_lsn >= lsn): break time.sleep(0.01) def version(self): p = subprocess.Popen([self.binary, "--version"], cwd=self.vardir, stdout=subprocess.PIPE) version = p.stdout.read().rstrip() p.wait() return version def _start_and_exit(self, args, gdb=None, valgrind=None): if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind if self.valgrind: with daemon.DaemonContext(working_directory=self.vardir): subprocess.check_call(args) else: if not self.gdb: args.append("--background") else: raise RuntimeError( "'--gdb' and '--start-and-exit' can't be defined together") self.server = subprocess.Popen(args, cwd=self.vardir) self.server.wait() def start(self, start_and_exit=None, gdb=None, valgrind=None, silent=True): if start_and_exit != None: self.start_and_exit = start_and_exit if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind self.debug = self.test_debug() if self.is_started: if not silent: print "The server is already started." return if not silent: print "Starting the server..." version = self.version() print "Starting {0} {1}.".format(os.path.basename(self.binary), version) check_port(self.port) args = self.prepare_args() if self.gdb: args = prepare_gdb(self.binary, args) print "You started the server in gdb mode." print "To attach, use `screen -r tnt-gdb`" elif self.valgrind: args = prepare_valgrind([self.binary] + args, self.valgrind_log, os.path.abspath( os.path.join( self.vardir, self.default_suppression_name))) else: args = [self.binary] + args if self.start_and_exit: self._start_and_exit(args) return self.process = subprocess.Popen(args, cwd=self.vardir) # wait until the server is connected self.wait_until_started() # Set is_started flag, to nicely support cleanup during an exception. self.is_started = True def stop(self, silent=True): """Stop server instance. Do nothing if the server is not started, to properly shut down the server in case of an exception during start up.""" if not self.is_started: if not silent: print "The server is not started." return if not silent: print "Stopping the server..." if self.process == None: self.kill_old_server() return # kill process pid = self.read_pidfile() if pid != -1: os.kill(pid, signal.SIGTERM) #self.process.kill(signal.SIGTERM) if self.gdb or self.valgrind: time = 0 while time < (1 << 30): if self.process.poll() != None: break time += 1 sleep(1) else: self.process.wait() self.wait_until_stopped() # clean-up processs flags self.is_started = False self.process = None def deploy(self, config=None, binary=None, vardir=None, mem=None, start_and_exit=None, gdb=None, valgrind=None, valgrind_sup=None, init_lua=None, silent=True, need_init=True): if config != None: self.config = config if binary != None: self.binary = binary if vardir != None: self.vardir = vardir if mem != None: self.mem = mem if start_and_exit != None: self.start_and_exit = start_and_exit if gdb != None: self.gdb = gdb if valgrind != None: self.valgrind = valgrind if init_lua != None: self.init_lua = os.path.abspath(init_lua) else: self.init_lua = None self.configure(self.config) self.install(self.binary, self.vardir, self.mem, silent) if need_init: self.init() self.start(self.start_and_exit, self.gdb, self.valgrind, silent) def restart(self): self.stop(silent=True) self.start(silent=True) def test_option_get(self, show, option_list_str): args = [self.binary] + option_list_str.split() if show: print " ".join([os.path.basename(self.binary)] + args[1:]) output = subprocess.Popen(args, cwd=self.vardir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() return output def test_option(self, option_list_str): print self.test_option_get(True, option_list_str) def test_debug(self): output = self.test_option_get(False, "-V") if re.search("-Debug", output): return True return False def kill_old_server(self, silent=True): """Kill old server instance if it exists.""" pid = self.read_pidfile() if pid == -1: return # Nothing to do if not silent: print " Found old server, pid {0}, killing...".format(pid) try: os.kill(pid, signal.SIGTERM) while os.kill(pid, 0) != -1: time.sleep(0.001) except OSError: pass def read_pidfile(self): if os.access(self.pidfile, os.F_OK) == False: # file is inaccessible (not exist or permission denied) return -1 pid = -1 try: with open(self.pidfile) as f: pid = int(f.read()) except: pass return pid def wait_until_started(self): """Wait until the server is started and accepting connections""" while self.read_pidfile() == -1: time.sleep(0.001) is_connected = False while not is_connected: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(("localhost", self.port)) is_connected = True sock.close() except socket.error as e: time.sleep(0.001) def wait_until_stopped(self): """Wait until the server is stoped and has closed sockets""" while self.read_pidfile() != -1: time.sleep(0.001) is_connected = False while not is_connected: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(("localhost", self.port)) is_connected = True sock.close() time.sleep(0.001) continue except socket.error as e: break def find_tests(self, test_suite, suite_path): def patterned(test, patterns): for i in patterns: if test.name.find(i) != -1: return True return False tests = [ PythonTest(k, test_suite.args, test_suite.ini) for k in sorted(glob.glob(os.path.join(suite_path, "*.test.py"))) ] tests += [ LuaTest(k, test_suite.args, test_suite.ini) for k in sorted(glob.glob(os.path.join(suite_path, "*.test.lua"))) ] test_suite.tests = filter( (lambda x: patterned(x, test_suite.args.tests)), tests)