def do_home_cleanup(self): if self.testbed().recovering: # WOW - not now return self._logger.info("Cleaning up home on %s", self.hostname) cmds = [ "find . -maxdepth 1 \( -name '.cache' -o -name '.local' -o -name '.config' -o -name 'nepi-*' \) -execdir rm -rf {} + " ] for cmd in cmds: (out, err), proc = server.popen_ssh_command( # Some apps need two kills cmd, host=self.hostip, port=None, user=self.slicename, agent=None, ident_key=self.ident_path, server_key=self.server_key, tty=True, # so that ps -N -T works as advertised... timeout=60, retry=3) proc.wait()
def do_proc_cleanup(self): if self.testbed().recovering: # WOW - not now return self._logger.info("Cleaning up processes on %s", self.hostname) cmds = [ "sudo -S killall python tcpdump || /bin/true ; " "sudo -S killall python tcpdump || /bin/true ; " "sudo -S kill $(ps -N -T -o pid --no-heading | grep -v $PPID | sort) || /bin/true ", "sudo -S killall -u %(slicename)s || /bin/true ", "sudo -S killall -u root || /bin/true ", "sudo -S killall -u %(slicename)s || /bin/true ", "sudo -S killall -u root || /bin/true ", ] for cmd in cmds: (out, err), proc = server.popen_ssh_command( # Some apps need two kills cmd % { 'slicename': self.slicename, }, host=self.hostip, port=None, user=self.slicename, agent=None, ident_key=self.ident_path, server_key=self.server_key, tty=True, # so that ps -N -T works as advertised... timeout=60, retry=3) proc.wait()
def do_proc_cleanup(self): if self.testbed().recovering: # WOW - not now return self._logger.info("Cleaning up processes on %s", self.hostname) cmds = [ "sudo -S killall python tcpdump || /bin/true ; " "sudo -S killall python tcpdump || /bin/true ; " "sudo -S kill $(ps -N -T -o pid --no-heading | grep -v $PPID | sort) || /bin/true ", "sudo -S killall -u %(slicename)s || /bin/true ", "sudo -S killall -u root || /bin/true ", "sudo -S killall -u %(slicename)s || /bin/true ", "sudo -S killall -u root || /bin/true ", ] for cmd in cmds: (out,err),proc = server.popen_ssh_command( # Some apps need two kills cmd % { 'slicename' : self.slicename , }, host = self.hostip, port = None, user = self.slicename, agent = None, ident_key = self.ident_path, server_key = self.server_key, tty = True, # so that ps -N -T works as advertised... timeout = 60, retry = 3 ) proc.wait()
def do_home_cleanup(self): if self.testbed().recovering: # WOW - not now return self._logger.info("Cleaning up home on %s", self.hostname) cmds = [ "find . -maxdepth 1 \( -name '.cache' -o -name '.local' -o -name '.config' -o -name 'nepi-*' \) -execdir rm -rf {} + " ] for cmd in cmds: (out,err),proc = server.popen_ssh_command( # Some apps need two kills cmd, host = self.hostip, port = None, user = self.slicename, agent = None, ident_key = self.ident_path, server_key = self.server_key, tty = True, # so that ps -N -T works as advertised... timeout = 60, retry = 3 ) proc.wait()
def sync_trace(self, local_dir, whichtrace): if whichtrace != 'netpipeStats': raise ValueError, "Unsupported trace %s" % (whichtrace,) local_path = os.path.join(local_dir, "netpipe_stats_%s" % (self.mode,)) # create parent local folders proc = subprocess.Popen( ["mkdir", "-p", os.path.dirname(local_path)], stdout = open("/dev/null","w"), stdin = open("/dev/null","r")) if proc.wait(): raise RuntimeError, "Failed to synchronize trace: %s %s" % (out,err,) (out,err),proc = server.popen_ssh_command( "echo 'Rules:' ; sudo -S netconfig show rules ; echo 'Pipes:' ; sudo -S netconfig show pipes", host = self.node.hostname, port = None, user = self.node.slicename, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key ) if proc.wait(): raise RuntimeError, "Failed to synchronize trace: %s %s" % (out,err,) # dump results to file f = open(local_path, "wb") f.write(err or "") f.write(out or "") f.close() return local_path
def remote_status(pid, ppid, host=None, port=None, user=None, agent=None, ident_key=None, server_key=None, hostip=None): """ Check the status of a process spawned with remote_spawn. Parameters: pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: One of NOT_STARTED, RUNNING, FINISHED """ (out, err), proc = server.popen_ssh_command( # Check only by pid. pid+ppid does not always work (especially with sudo) " (( ps --pid %(pid)d -o pid | grep -c %(pid)d && echo 'wait') || echo 'done' ) | tail -n 1" % { 'ppid': ppid, 'pid': pid, }, host=host, port=port, user=user, agent=agent, ident_key=ident_key, server_key=server_key, hostip=hostip) if proc.wait(): return NOT_STARTED status = False if err: if err.strip().find("Error, do this: mount -t proc none /proc") >= 0: status = True elif out: status = (out.strip() == 'wait') else: return NOT_STARTED return RUNNING if status else FINISHED
def _popen_ssh_command(self, command, retry = 0, noerrors=False, timeout=None): (out,err),proc = server.popen_ssh_command( command, host = self.node.hostname, port = None, user = self.node.slicename, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key, timeout = timeout, retry = retry ) if server.eintr_retry(proc.wait)(): if not noerrors: raise RuntimeError, (out, err) return (out, err), proc
def refresh(self): if self.configured: # refresh rule scope, options = self._get_ruledef() command = "sudo -S netconfig refresh %s %s %s" % (self.mode, scope, options) (out,err),proc = server.popen_ssh_command( command, host = self.node.hostname, port = None, user = self.node.slicename, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key ) if proc.wait(): raise RuntimeError, "Failed instal build sources: %s %s" % (out,err,)
def apply_route_rules(self, rules, method): (out, err), proc = server.popen_ssh_command( "( sudo -S bash -c 'cat /vsys/%(method)s.out >&2' & ) ; sudo -S bash -c 'cat > /vsys/%(method)s.in' ; sleep 0.5" % dict(home=server.shell_escape(self.home_path), method=method), host=self.hostip, port=None, user=self.slicename, agent=None, ident_key=self.ident_path, server_key=self.server_key, stdin='\n'.join(rules), timeout=300) if proc.wait() or err: raise RuntimeError, "Could not set routes (%s) errors: %s%s" % ( rules, out, err) elif out or err: logger.debug("%s said: %s%s", method, out, err)
def remote_status(pid, ppid, host = None, port = None, user = None, agent = None, ident_key = None, server_key = None, hostip = None): """ Check the status of a process spawned with remote_spawn. Parameters: pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: One of NOT_STARTED, RUNNING, FINISHED """ (out,err),proc = server.popen_ssh_command( # Check only by pid. pid+ppid does not always work (especially with sudo) " (( ps --pid %(pid)d -o pid | grep -c %(pid)d && echo 'wait') || echo 'done' ) | tail -n 1" % { 'ppid' : ppid, 'pid' : pid, }, host = host, port = port, user = user, agent = agent, ident_key = ident_key, server_key = server_key, hostip = hostip ) if proc.wait(): return NOT_STARTED status = False if err: if err.strip().find("Error, do this: mount -t proc none /proc") >= 0: status = True elif out: status = (out.strip() == 'wait') else: return NOT_STARTED return RUNNING if status else FINISHED
def apply_route_rules(self, rules, method): (out,err),proc = server.popen_ssh_command( "( sudo -S bash -c 'cat /vsys/%(method)s.out >&2' & ) ; sudo -S bash -c 'cat > /vsys/%(method)s.in' ; sleep 0.5" % dict( home = server.shell_escape(self.home_path), method = method), host = self.hostip, port = None, user = self.slicename, agent = None, ident_key = self.ident_path, server_key = self.server_key, stdin = '\n'.join(rules), timeout = 300 ) if proc.wait() or err: raise RuntimeError, "Could not set routes (%s) errors: %s%s" % (rules,out,err) elif out or err: logger.debug("%s said: %s%s", method, out, err)
def install_dependencies(self): if self.required_packages and not self._installed: # If we need rpmfusion, we must install the repo definition and the gpg keys if self.rpmFusion: if self.operatingSystem == 'f12': # Fedora 12 requires a different rpmfusion package RPM_FUSION_URL = self.RPM_FUSION_URL_F12 else: # This one works for f13+ RPM_FUSION_URL = self.RPM_FUSION_URL rpmFusion = ( 'rpm -q rpmfusion-free-release || sudo -S rpm -i %(RPM_FUSION_URL)s' ) % { 'RPM_FUSION_URL': RPM_FUSION_URL } else: rpmFusion = '' if rpmFusion: (out, err), proc = server.popen_ssh_command( rpmFusion, host=self.hostip, port=None, user=self.slicename, agent=None, ident_key=self.ident_path, server_key=self.server_key, timeout=600, ) if proc.wait(): if self.check_bad_host(out, err): self.blacklist() raise RuntimeError, "Failed to set up application on host %s: %s %s" % ( self.hostname, out, err, ) # Launch p2p yum dependency installer self._yum_dependencies.async_setup()
def remote_check_pid(pidfile, host=None, port=None, user=None, agent=None, ident_key=None, server_key=None, hostip=None): """ Check the pidfile of a process spawned with remote_spawn. Parameters: pidfile: the pidfile passed to remote_span host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: A (pid, ppid) tuple useful for calling remote_status and remote_kill, or None if the pidfile isn't valid yet (maybe the process is still starting). """ (out, err), proc = server.popen_ssh_command("cat %(pidfile)s" % { 'pidfile': pidfile, }, host=host, port=port, user=user, agent=agent, ident_key=ident_key, server_key=server_key, hostip=hostip) if proc.wait(): return None if out: try: return map(int, out.strip().split(' ', 1)) except: # Ignore, many ways to fail that don't matter that much return None
def _popen_ssh_command(self, command, retry=0, noerrors=False, timeout=None): (out, err), proc = server.popen_ssh_command(command, host=self.node.hostname, port=None, user=self.node.slicename, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key, timeout=timeout, retry=retry) if server.eintr_retry(proc.wait)(): if not noerrors: raise RuntimeError, (out, err) return (out, err), proc
def configure(self): # set up rule scope, options = self._get_ruledef() command = "sudo -S netconfig config %s %s %s" % (self.mode, scope, options) (out,err),proc = server.popen_ssh_command( command, host = self.node.hostname, port = None, user = self.node.slicename, agent = None, ident_key = self.node.ident_path, server_key = self.node.server_key ) if proc.wait(): raise RuntimeError, "Failed instal build sources: %s %s" % (out,err,) # we have to clean up afterwards self.configured = True
def sync_trace(self, local_dir, whichtrace): if whichtrace != 'netpipeStats': raise ValueError, "Unsupported trace %s" % (whichtrace, ) local_path = os.path.join(local_dir, "netpipe_stats_%s" % (self.mode, )) # create parent local folders proc = subprocess.Popen( ["mkdir", "-p", os.path.dirname(local_path)], stdout=open("/dev/null", "w"), stdin=open("/dev/null", "r")) if proc.wait(): raise RuntimeError, "Failed to synchronize trace: %s %s" % ( out, err, ) (out, err), proc = server.popen_ssh_command( "echo 'Rules:' ; sudo -S netconfig show rules ; echo 'Pipes:' ; sudo -S netconfig show pipes", host=self.node.hostname, port=None, user=self.node.slicename, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key) if proc.wait(): raise RuntimeError, "Failed to synchronize trace: %s %s" % ( out, err, ) # dump results to file f = open(local_path, "wb") f.write(err or "") f.write(out or "") f.close() return local_path
def refresh(self): if self.configured: # refresh rule scope, options = self._get_ruledef() command = "sudo -S netconfig refresh %s %s %s" % (self.mode, scope, options) (out, err), proc = server.popen_ssh_command( command, host=self.node.hostname, port=None, user=self.node.slicename, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key) if proc.wait(): raise RuntimeError, "Failed instal build sources: %s %s" % ( out, err, )
def remote_check_pid(pidfile, host = None, port = None, user = None, agent = None, ident_key = None, server_key = None, hostip = None): """ Check the pidfile of a process spawned with remote_spawn. Parameters: pidfile: the pidfile passed to remote_span host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: A (pid, ppid) tuple useful for calling remote_status and remote_kill, or None if the pidfile isn't valid yet (maybe the process is still starting). """ (out,err),proc = server.popen_ssh_command( "cat %(pidfile)s" % { 'pidfile' : pidfile, }, host = host, port = port, user = user, agent = agent, ident_key = ident_key, server_key = server_key, hostip = hostip ) if proc.wait(): return None if out: try: return map(int,out.strip().split(' ',1)) except: # Ignore, many ways to fail that don't matter that much return None
def install_dependencies(self): if self.required_packages and not self._installed: # If we need rpmfusion, we must install the repo definition and the gpg keys if self.rpmFusion: if self.operatingSystem == 'f12': # Fedora 12 requires a different rpmfusion package RPM_FUSION_URL = self.RPM_FUSION_URL_F12 else: # This one works for f13+ RPM_FUSION_URL = self.RPM_FUSION_URL rpmFusion = ( 'rpm -q rpmfusion-free-release || sudo -S rpm -i %(RPM_FUSION_URL)s' ) % { 'RPM_FUSION_URL' : RPM_FUSION_URL } else: rpmFusion = '' if rpmFusion: (out,err),proc = server.popen_ssh_command( rpmFusion, host = self.hostip, port = None, user = self.slicename, agent = None, ident_key = self.ident_path, server_key = self.server_key, timeout = 600, ) if proc.wait(): if self.check_bad_host(out,err): self.blacklist() raise RuntimeError, "Failed to set up application on host %s: %s %s" % (self.hostname, out,err,) # Launch p2p yum dependency installer self._yum_dependencies.async_setup()
def configure(self): # set up rule scope, options = self._get_ruledef() command = "sudo -S netconfig config %s %s %s" % (self.mode, scope, options) (out, err), proc = server.popen_ssh_command(command, host=self.node.hostname, port=None, user=self.node.slicename, agent=None, ident_key=self.node.ident_path, server_key=self.node.server_key) if proc.wait(): raise RuntimeError, "Failed instal build sources: %s %s" % ( out, err, ) # we have to clean up afterwards self.configured = True
def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/dev/null', home=None, create_home=False, sudo=False, host = None, port = None, user = None, agent = None, ident_key = None, server_key = None, tty = False, hostip = None): """ Spawn a remote command such that it will continue working asynchronously. Parameters: command: the command to run - it should be a single line. pidfile: path of a (ideally unique to this task) pidfile for tracking the process. stdout: path of a file to redirect standard output to - must be a string. Defaults to /dev/null stderr: path of a file to redirect standard error to - string or the special STDOUT value to redirect to the same file stdout was redirected to. Defaults to STDOUT. stdin: path of a file with input to be piped into the command's standard input home: path of a folder to use as working directory - should exist, unless you specify create_home create_home: if True, the home folder will be created first with mkdir -p sudo: whether the command needs to be executed as root host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: (stdout, stderr), process Of the spawning process, which only captures errors at spawning time. Usually only useful for diagnostics. """ # Start process in a "daemonized" way, using nohup and heavy # stdin/out redirection to avoid connection issues if stderr is STDOUT: stderr = '&1' else: stderr = ' ' + stderr daemon_command = '{ { %(command)s > %(stdout)s 2>%(stderr)s < %(stdin)s & } ; echo $! 1 > %(pidfile)s ; }' % { 'command' : command, 'pidfile' : server.shell_escape(pidfile), 'stdout' : stdout, 'stderr' : stderr, 'stdin' : stdin, } cmd = "%(create)s%(gohome)s rm -f %(pidfile)s ; %(sudo)s nohup bash -c %(command)s " % { 'command' : server.shell_escape(daemon_command), 'sudo' : 'sudo -S' if sudo else '', 'pidfile' : server.shell_escape(pidfile), 'gohome' : 'cd %s ; ' % (server.shell_escape(home),) if home else '', 'create' : 'mkdir -p %s ; ' % (server.shell_escape,) if create_home else '', } (out,err),proc = server.popen_ssh_command( cmd, host = host, port = port, user = user, agent = agent, ident_key = ident_key, server_key = server_key, tty = tty , hostip = hostip ) if proc.wait(): raise RuntimeError, "Failed to set up application on host %s: %s %s" % (host, out,err,) return (out,err),proc
def _install_scripts(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Install the tun_connect script and tunalloc utility from nepi.util import tunchannel from nepi.util import ipaddr2 sources = [ os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'), os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'), re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific ] if local.filter_module: filter_sources = filter(bool,map(str.strip,local.filter_module.module.split())) filter_module = filter_sources[0] # Translate paths to builtin sources for i,source in enumerate(filter_sources): if not os.path.exists(source): # Um... try the builtin folder source = os.path.join(os.path.dirname(__file__), "scripts", source) if os.path.exists(source): # Yep... replace filter_sources[i] = source sources.extend(set(filter_sources)) else: filter_module = None filter_sources = None dest = "%s@%s:%s" % ( local.node.slicename, local.node.hostname, os.path.join(self.home_path,'.'),) (out,err),proc = server.eintr_retry(server.popen_scp)( sources, dest, ident_key = local.node.ident_path, server_key = local.node.server_key ) if proc.wait(): raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (sources, out,err,) # Make sure all dependencies are satisfied local.node.wait_dependencies() cmd = ( ( "cd %(home)s && " "gcc -fPIC -shared tunalloc.c -o tunalloc.so && " "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && " "mkdir -p python-iovec && " "cd python-iovec && " "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. && " "cd .. " + ( " && " "gcc -fPIC -shared %(sources)s -o %(module)s.so " % { 'module' : os.path.basename(filter_module).rsplit('.',1)[0], 'sources' : ' '.join(map(os.path.basename,filter_sources)) } if filter_module is not None and filter_module.endswith('.c') else "" ) + ( " && " "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && " "mkdir -p python-passfd && " "cd python-passfd && " "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. " if local.tun_proto == "fd" else "" ) ) % { 'home' : server.shell_escape(self.home_path), 'passfd_url' : "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz", 'iovec_url' : "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz", } ) (out,err),proc = server.popen_ssh_command( cmd, host = local.node.hostname, port = None, user = local.node.slicename, agent = None, ident_key = local.node.ident_path, server_key = local.node.server_key, timeout = 300 ) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)
def _install_scripts(self): local = self.local() if not local: raise RuntimeError, "Lost reference to peering interfaces before launching" if not local.node: raise RuntimeError, "Unconnected TUN - missing node" # Install the tun_connect script and tunalloc utility from nepi.util import tunchannel from nepi.util import ipaddr2 sources = [ os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'), os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'), re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific ] if local.filter_module: filter_sources = filter( bool, map(str.strip, local.filter_module.module.split())) filter_module = filter_sources[0] # Translate paths to builtin sources for i, source in enumerate(filter_sources): if not os.path.exists(source): # Um... try the builtin folder source = os.path.join(os.path.dirname(__file__), "scripts", source) if os.path.exists(source): # Yep... replace filter_sources[i] = source sources.extend(set(filter_sources)) else: filter_module = None filter_sources = None dest = "%s@%s:%s" % ( local.node.slicename, local.node.hostname, os.path.join(self.home_path, '.'), ) (out, err), proc = server.eintr_retry(server.popen_scp)( sources, dest, ident_key=local.node.ident_path, server_key=local.node.server_key) if proc.wait(): raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % ( sources, out, err, ) # Make sure all dependencies are satisfied local.node.wait_dependencies() cmd = ( ("cd %(home)s && " "gcc -fPIC -shared tunalloc.c -o tunalloc.so && " "wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && " "mkdir -p python-iovec && " "cd python-iovec && " "tar xzf ../python-iovec-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. && " "cd .. " + (" && " "gcc -fPIC -shared %(sources)s -o %(module)s.so " % { 'module': os.path.basename(filter_module).rsplit('.', 1)[0], 'sources': ' '.join(map(os.path.basename, filter_sources)) } if filter_module is not None and filter_module.endswith('.c') else "") + (" && " "wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && " "mkdir -p python-passfd && " "cd python-passfd && " "tar xzf ../python-passfd-src.tar.gz --strip-components=1 && " "python setup.py build && " "python setup.py install --install-lib .. " if local.tun_proto == "fd" else "")) % { 'home': server.shell_escape(self.home_path), 'passfd_url': "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz", 'iovec_url': "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz", }) (out, err), proc = server.popen_ssh_command( cmd, host=local.node.hostname, port=None, user=local.node.slicename, agent=None, ident_key=local.node.ident_path, server_key=local.node.server_key, timeout=300) if proc.wait(): raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % ( out, err, )
def remote_kill(pid, ppid, sudo=False, host=None, port=None, user=None, agent=None, ident_key=None, server_key=None, hostip=None, nowait=False): """ Kill a process spawned with remote_spawn. First tries a SIGTERM, and if the process does not end in 10 seconds, it sends a SIGKILL. Parameters: pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid sudo: whether the command was run with sudo - careful killing like this. host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: Nothing, should have killed the process """ if sudo: subkill = "$(ps --ppid %(pid)d -o pid h)" % {'pid': pid} else: subkill = "" cmd = """ SUBKILL="%(subkill)s" ; %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill %(pid)d $SUBKILL || /bin/true for x in 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 ; do sleep 0.2 if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` == '0' ]; then break else %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill %(pid)d $SUBKILL || /bin/true fi sleep 1.8 done if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` != '0' ]; then %(sudo)s kill -9 -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill -9 %(pid)d $SUBKILL || /bin/true fi """ if nowait: cmd = "( %s ) >/dev/null 2>/dev/null </dev/null &" % (cmd, ) (out, err), proc = server.popen_ssh_command(cmd % { 'ppid': ppid, 'pid': pid, 'sudo': 'sudo -S' if sudo else '', 'subkill': subkill, }, host=host, port=port, user=user, agent=agent, ident_key=ident_key, server_key=server_key, hostip=hostip) # wait, don't leave zombies around proc.wait()
def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/dev/null', home=None, create_home=False, sudo=False, host=None, port=None, user=None, agent=None, ident_key=None, server_key=None, tty=False, hostip=None): """ Spawn a remote command such that it will continue working asynchronously. Parameters: command: the command to run - it should be a single line. pidfile: path of a (ideally unique to this task) pidfile for tracking the process. stdout: path of a file to redirect standard output to - must be a string. Defaults to /dev/null stderr: path of a file to redirect standard error to - string or the special STDOUT value to redirect to the same file stdout was redirected to. Defaults to STDOUT. stdin: path of a file with input to be piped into the command's standard input home: path of a folder to use as working directory - should exist, unless you specify create_home create_home: if True, the home folder will be created first with mkdir -p sudo: whether the command needs to be executed as root host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: (stdout, stderr), process Of the spawning process, which only captures errors at spawning time. Usually only useful for diagnostics. """ # Start process in a "daemonized" way, using nohup and heavy # stdin/out redirection to avoid connection issues if stderr is STDOUT: stderr = '&1' else: stderr = ' ' + stderr daemon_command = '{ { %(command)s > %(stdout)s 2>%(stderr)s < %(stdin)s & } ; echo $! 1 > %(pidfile)s ; }' % { 'command': command, 'pidfile': server.shell_escape(pidfile), 'stdout': stdout, 'stderr': stderr, 'stdin': stdin, } cmd = "%(create)s%(gohome)s rm -f %(pidfile)s ; %(sudo)s nohup bash -c %(command)s " % { 'command': server.shell_escape(daemon_command), 'sudo': 'sudo -S' if sudo else '', 'pidfile': server.shell_escape(pidfile), 'gohome': 'cd %s ; ' % (server.shell_escape(home), ) if home else '', 'create': 'mkdir -p %s ; ' % (server.shell_escape, ) if create_home else '', } (out, err), proc = server.popen_ssh_command(cmd, host=host, port=port, user=user, agent=agent, ident_key=ident_key, server_key=server_key, tty=tty, hostip=hostip) if proc.wait(): raise RuntimeError, "Failed to set up application on host %s: %s %s" % ( host, out, err, ) return (out, err), proc
def remote_kill(pid, ppid, sudo = False, host = None, port = None, user = None, agent = None, ident_key = None, server_key = None, hostip = None, nowait = False): """ Kill a process spawned with remote_spawn. First tries a SIGTERM, and if the process does not end in 10 seconds, it sends a SIGKILL. Parameters: pid/ppid: pid and parent-pid of the spawned process. See remote_check_pid sudo: whether the command was run with sudo - careful killing like this. host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command Returns: Nothing, should have killed the process """ if sudo: subkill = "$(ps --ppid %(pid)d -o pid h)" % { 'pid' : pid } else: subkill = "" cmd = """ SUBKILL="%(subkill)s" ; %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill %(pid)d $SUBKILL || /bin/true for x in 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 ; do sleep 0.2 if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` == '0' ]; then break else %(sudo)s kill -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill %(pid)d $SUBKILL || /bin/true fi sleep 1.8 done if [ `ps --pid %(pid)d -o pid | grep -c %(pid)d` != '0' ]; then %(sudo)s kill -9 -- -%(pid)d $SUBKILL || /bin/true %(sudo)s kill -9 %(pid)d $SUBKILL || /bin/true fi """ if nowait: cmd = "( %s ) >/dev/null 2>/dev/null </dev/null &" % (cmd,) (out,err),proc = server.popen_ssh_command( cmd % { 'ppid' : ppid, 'pid' : pid, 'sudo' : 'sudo -S' if sudo else '', 'subkill' : subkill, }, host = host, port = port, user = user, agent = agent, ident_key = ident_key, server_key = server_key, hostip = hostip ) # wait, don't leave zombies around proc.wait()