def start(self):
self._logger.info("Starting %s", self)
# Create shell script with the command
# This way, complex commands and scripts can be ran seamlessly
# sync files
command = cStringIO.StringIO()
command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (
':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
))
command.write('export PATH=$PATH:%s\n' % (
':'.join(["${HOME}/"+server.shell_escape(s) for s in self.node.pythonpath])
))
if self.node.env:
for envkey, envvals in self.node.env.iteritems():
for envval in envvals:
command.write('export %s=%s\n' % (envkey, envval))
command.write(self.command)
command.seek(0)
try:
self._popen_scp(
command,
'%s@%s:%s' % (self.node.slicename, self.node.hostname,
os.path.join(self.home_path, "app.sh"))
)
except RuntimeError, e:
raise RuntimeError, "Failed to set up application: %s %s" \
% (e.args[0], e.args[1],)
def start(self):
self._logger.info("Starting %s", self)
# Create shell script with the command
# This way, complex commands and scripts can be ran seamlessly
# sync files
command = cStringIO.StringIO()
command.write('export PYTHONPATH=$PYTHONPATH:%s\n' % (':'.join([
"${HOME}/" + server.shell_escape(s) for s in self.node.pythonpath
])))
command.write('export PATH=$PATH:%s\n' % (':'.join([
"${HOME}/" + server.shell_escape(s) for s in self.node.pythonpath
])))
if self.node.env:
for envkey, envvals in self.node.env.iteritems():
for envval in envvals:
command.write('export %s=%s\n' % (envkey, envval))
command.write(self.command)
command.seek(0)
try:
self._popen_scp(
command, '%s@%s:%s' % (self.node.slicename, self.node.hostname,
os.path.join(self.home_path, "app.sh")))
except RuntimeError, e:
raise RuntimeError, "Failed to set up application: %s %s" \
% (e.args[0], e.args[1],)
def wait(self):
local = self.local()
# Wait for the connection to be established
retrytime = 2.0
for spin in xrange(30):
if self.status() != rspawn.RUNNING:
self._logger.warn("FAILED TO CONNECT! %s", self)
break
# Connected?
(out, err), proc = server.eintr_retry(server.popen_ssh_command)(
"cd %(home)s ; grep -a -c Connected capture" %
dict(home=server.shell_escape(self.home_path)),
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=60,
err_on_timeout=False)
proc.wait()
if out.strip() == '1':
break
# At least listening?
(out, err), proc = server.eintr_retry(server.popen_ssh_command)(
"cd %(home)s ; grep -a -c Listening capture" %
dict(home=server.shell_escape(self.home_path)),
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=60,
err_on_timeout=False)
proc.wait()
time.sleep(min(30.0, retrytime))
retrytime *= 1.1
else:
(out, err), proc = server.eintr_retry(server.popen_ssh_command)(
"cat %(home)s/capture" %
dict(home=server.shell_escape(self.home_path)),
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=60,
retry=3,
err_on_timeout=False)
proc.wait()
raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self, out, err)
def _replace_paths(self, command):
"""
Replace all special path tags with shell-escaped actual paths.
"""
# need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
root = '' if self.home_path.startswith('/') else "${HOME}/"
return ( command
.replace("${SOURCES}", root+server.shell_escape(self.home_path))
.replace("${BUILD}", root+server.shell_escape(os.path.join(self.home_path,'build'))) )
def _replace_paths(self, command):
"""
Replace all special path tags with shell-escaped actual paths.
"""
# need to append ${HOME} if paths aren't absolute, to MAKE them absolute.
root = '' if self.home_path.startswith('/') else "${HOME}/"
return (command.replace(
"${SOURCES}", root + server.shell_escape(self.home_path)).replace(
"${BUILD}", root +
server.shell_escape(os.path.join(self.home_path, 'build'))))
def _nepi_testbed_environment_setup_get(self):
command = cStringIO.StringIO()
command.write('export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(
["${HOME}/" + server.shell_escape(s) for s in self.pythonpath])))
command.write(' ; export PATH=$PATH:%s' % (':'.join(
["${HOME}/" + server.shell_escape(s) for s in self.pythonpath])))
if self.env:
for envkey, envvals in self.env.iteritems():
for envval in envvals:
command.write(' ; export %s=%s' % (envkey, envval))
return command.getvalue()
def _nepi_testbed_environment_setup_get(self):
command = cStringIO.StringIO()
command.write('export PYTHONPATH=$PYTHONPATH:%s' % (
':'.join(["${HOME}/"+server.shell_escape(s) for s in self.pythonpath])
))
command.write(' ; export PATH=$PATH:%s' % (
':'.join(["${HOME}/"+server.shell_escape(s) for s in self.pythonpath])
))
if self.env:
for envkey, envvals in self.env.iteritems():
for envval in envvals:
command.write(' ; export %s=%s' % (envkey, envval))
return command.getvalue()
def _make_home(self):
local = self.local()
if not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
if not local.node:
raise RuntimeError, "Unconnected TUN - missing node"
# Make sure all the paths are created where
# they have to be created for deployment
# Also remove pidfile, if there is one.
# Old pidfiles from previous runs can be troublesome.
cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % {
'home' : server.shell_escape(self.home_path)
}
(out,err),proc = server.eintr_retry(server.popen_ssh_command)(
cmd,
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 60,
retry = 3
)
if proc.wait():
raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)
def _make_home(self):
local = self.local()
if not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
if not local.node:
raise RuntimeError, "Unconnected TUN - missing node"
# Make sure all the paths are created where
# they have to be created for deployment
# Also remove pidfile, if there is one.
# Old pidfiles from previous runs can be troublesome.
cmd = "mkdir -p %(home)s ; rm -f %(home)s/pid %(home)s/*.so" % {
'home': server.shell_escape(self.home_path)
}
(out, err), proc = server.eintr_retry(server.popen_ssh_command)(
cmd,
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=60,
retry=3)
if proc.wait():
raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (
out,
err,
)
def __init__(self, api=None):
super(CCNxDaemon, self).__init__(api)
# Attributes
self.ccnLocalPort = None
self.ccnRoutes = None
self.ccnxVersion = "0.7.1"
self.repository = False
#self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
self.ccnx_sources = "http://www.ccnx.org/releases/ccnx-%s.tar.gz"
self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'
self.ccnx_build = (
" ( "
" cd .. && "
" test -d ccnx-src/build/bin "
" ) || ( "
# Not working, rebuild
"("
" mkdir -p ccnx-src && "
" wget -q -c -O ccnx-src.tar.gz %(ccnx_source_url)s &&"
" tar xf ccnx-src.tar.gz --strip-components=1 -C ccnx-src "
") && "
"cd ccnx-src && "
"./configure && make"
" )") % dict(ccnx_source_url=server.shell_escape(
self.ccnx_sources % self.ccnxVersion), )
self.ccnx_install = (" ( "
" test -d ${BUILD}/ccnx-src/bin && "
" cp -r ${BUILD}/ccnx-src/bin ${SOURCES}"
" )")
self.env['PATH'] = "$PATH:${SOURCES}/bin"
def _do_launch_build(self):
script = "bash ./nepi-build.sh"
if self._master_passphrase:
script = "NEPI_MASTER_PASSPHRASE=%s %s" % (server.shell_escape(
self._master_passphrase), script)
(out, err), proc = rspawn.remote_spawn(
script,
pidfile='build-pid',
home=self.home_path,
stdin='/dev/null',
stdout='buildlog',
stderr=rspawn.STDOUT,
host=self.node.hostname,
port=None,
user=self.node.slicename,
agent=None,
ident_key=self.node.ident_path,
server_key=self.node.server_key,
hostip=self.node.hostip,
)
if proc.wait():
if self.check_bad_host(out, err):
self.node.blacklist()
raise RuntimeError, "Failed to set up build slave %s: %s %s" % (
self.home_path,
out,
err,
)
pid = ppid = None
delay = 1.0
for i in xrange(5):
pidtuple = rspawn.remote_check_pid(os.path.join(
self.home_path, 'build-pid'),
host=self.node.hostip,
port=None,
user=self.node.slicename,
agent=None,
ident_key=self.node.ident_path,
server_key=self.node.server_key,
hostip=self.node.hostip)
if pidtuple:
pid, ppid = pidtuple
self._build_pid, self._build_ppid = pidtuple
break
else:
time.sleep(delay)
delay = min(30, delay * 1.2)
else:
raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (
self.home_path, )
self._logger.info("Deploying %s at %s", self, self.node.hostname)
def _do_launch_build(self):
script = "bash ./nepi-build.sh"
if self._master_passphrase:
script = "NEPI_MASTER_PASSPHRASE=%s %s" % (
server.shell_escape(self._master_passphrase),
script
)
(out,err),proc = rspawn.remote_spawn(
script,
pidfile = 'build-pid',
home = self.home_path,
stdin = '/dev/null',
stdout = 'buildlog',
stderr = rspawn.STDOUT,
host = self.node.hostname,
port = None,
user = self.node.slicename,
agent = None,
ident_key = self.node.ident_path,
server_key = self.node.server_key,
hostip = self.node.hostip,
)
if proc.wait():
if self.check_bad_host(out, err):
self.node.blacklist()
raise RuntimeError, "Failed to set up build slave %s: %s %s" % (self.home_path, out,err,)
pid = ppid = None
delay = 1.0
for i in xrange(5):
pidtuple = rspawn.remote_check_pid(
os.path.join(self.home_path,'build-pid'),
host = self.node.hostip,
port = None,
user = self.node.slicename,
agent = None,
ident_key = self.node.ident_path,
server_key = self.node.server_key,
hostip = self.node.hostip
)
if pidtuple:
pid, ppid = pidtuple
self._build_pid, self._build_ppid = pidtuple
break
else:
time.sleep(delay)
delay = min(30,delay*1.2)
else:
raise RuntimeError, "Failed to set up build slave %s: cannot get pid" % (self.home_path,)
self._logger.info("Deploying %s at %s", self, self.node.hostname)
def _make_home(self):
# Make sure all the paths are created where
# they have to be created for deployment
# sync files
try:
self._popen_ssh_command(
"mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
% { 'home' : server.shell_escape(self.home_path) },
timeout = 120,
retry = 3
)
except RuntimeError, e:
raise RuntimeError, "Failed to set up application %s: %s %s" % (self.home_path, e.args[0], e.args[1],)
def _do_install(self):
if self.install:
self._logger.info("Installing %s at %s", self, self.node.hostname)
# Install application
try:
command = "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
{
'command' : self._replace_paths(self.install),
'home' : server.shell_escape(self.home_path),
}
self._popen_ssh_command(command)
except RuntimeError, e:
if self.check_bad_host(e.args[0], e.args[1]):
self.node.blacklist()
raise RuntimeError, "Failed install build sources on node %s: %s %s. COMMAND %s" % (
self.node.hostname, e.args[0], e.args[1], command)
def _make_home(self):
# Make sure all the paths are created where
# they have to be created for deployment
# sync files
try:
self._popen_ssh_command(
"mkdir -p %(home)s && ( rm -f %(home)s/{pid,build-pid,nepi-build.sh} >/dev/null 2>&1 || /bin/true )" \
% { 'home' : server.shell_escape(self.home_path) },
timeout = 120,
retry = 3
)
except RuntimeError, e:
raise RuntimeError, "Failed to set up application %s: %s %s" % (
self.home_path,
e.args[0],
e.args[1],
)
def _do_install(self):
if self.install:
self._logger.info("Installing %s at %s", self, self.node.hostname)
# Install application
try:
command = "cd %(home)s && cd build && ( %(command)s ) > ${HOME}/%(home)s/installlog 2>&1 || ( tail ${HOME}/%(home)s/{install,build}log >&2 && false )" % \
{
'command' : self._replace_paths(self.install),
'home' : server.shell_escape(self.home_path),
}
self._popen_ssh_command(command)
except RuntimeError, e:
if self.check_bad_host(e.args[0], e.args[1]):
self.node.blacklist()
raise RuntimeError, "Failed install build sources on node %s: %s %s. COMMAND %s" % (
self.node.hostname, e.args[0], e.args[1], command)
def apply_route_rules(self, rules, method):
(out, err), proc = server.popen_ssh_command(
"( sudo -S bash -c 'cat /vsys/%(method)s.out >&2' & ) ; sudo -S bash -c 'cat > /vsys/%(method)s.in' ; sleep 0.5"
% dict(home=server.shell_escape(self.home_path), method=method),
host=self.hostip,
port=None,
user=self.slicename,
agent=None,
ident_key=self.ident_path,
server_key=self.server_key,
stdin='\n'.join(rules),
timeout=300)
if proc.wait() or err:
raise RuntimeError, "Could not set routes (%s) errors: %s%s" % (
rules, out, err)
elif out or err:
logger.debug("%s said: %s%s", method, out, err)
def apply_route_rules(self, rules, method):
(out,err),proc = server.popen_ssh_command(
"( sudo -S bash -c 'cat /vsys/%(method)s.out >&2' & ) ; sudo -S bash -c 'cat > /vsys/%(method)s.in' ; sleep 0.5" % dict(
home = server.shell_escape(self.home_path),
method = method),
host = self.hostip,
port = None,
user = self.slicename,
agent = None,
ident_key = self.ident_path,
server_key = self.server_key,
stdin = '\n'.join(rules),
timeout = 300
)
if proc.wait() or err:
raise RuntimeError, "Could not set routes (%s) errors: %s%s" % (rules,out,err)
elif out or err:
logger.debug("%s said: %s%s", method, out, err)
def if_name(self):
if not self._if_name:
# Inspect the trace to check the assigned iface
local = self.local()
if local:
cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict(
home=server.shell_escape(self.home_path))
for spin in xrange(30):
(out,
err), proc = server.eintr_retry(server.popen_ssh_command)(
cmd,
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=60,
err_on_timeout=False)
if proc.wait():
self._logger.debug("if_name: failed cmd %s", cmd)
time.sleep(1)
continue
out = out.strip()
match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*", out)
if match:
self._if_name = match.group(1)
break
elif out:
self._logger.debug(
"if_name: %r does not match expected pattern from cmd %s",
out, cmd)
else:
self._logger.debug("if_name: empty output from cmd %s",
cmd)
time.sleep(3)
else:
self._logger.warn("if_name: Could not get interface name")
return self._if_name
def if_name(self):
if not self._if_name:
# Inspect the trace to check the assigned iface
local = self.local()
if local:
cmd = "cd %(home)s ; grep -a 'Using tun:' capture | head -1" % dict(
home = server.shell_escape(self.home_path))
for spin in xrange(30):
(out,err),proc = server.eintr_retry(server.popen_ssh_command)(
cmd,
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 60,
err_on_timeout = False
)
if proc.wait():
self._logger.debug("if_name: failed cmd %s", cmd)
time.sleep(1)
continue
out = out.strip()
match = re.match(r"Using +tun: +([-a-zA-Z0-9]*).*",out)
if match:
self._if_name = match.group(1)
break
elif out:
self._logger.debug("if_name: %r does not match expected pattern from cmd %s", out, cmd)
else:
self._logger.debug("if_name: empty output from cmd %s", cmd)
time.sleep(3)
else:
self._logger.warn("if_name: Could not get interface name")
return self._if_name
def __init__(self, api=None):
super(CCNxDaemon,self).__init__(api)
# Attributes
self.ccnLocalPort = None
self.ccnRoutes = None
self.ccnxVersion = "0.7.1"
self.repository = False
#self.ccnx_0_6_0_sources = "http://yans.pl.sophia.inria.fr/libs/ccnx-0.6.0.tar.gz"
self.ccnx_sources = "http://www.ccnx.org/releases/ccnx-%s.tar.gz"
self.buildDepends = 'make gcc openssl-devel expat-devel libpcap-devel libxml2-devel'
self.ccnx_build = (
" ( "
" cd .. && "
" test -d ccnx-src/build/bin "
" ) || ( "
# Not working, rebuild
"("
" mkdir -p ccnx-src && "
" wget -q -c -O ccnx-src.tar.gz %(ccnx_source_url)s &&"
" tar xf ccnx-src.tar.gz --strip-components=1 -C ccnx-src "
") && "
"cd ccnx-src && "
"./configure && make"
" )") % dict(
ccnx_source_url = server.shell_escape(self.ccnx_sources % self.ccnxVersion),
)
self.ccnx_install = (
" ( "
" test -d ${BUILD}/ccnx-src/bin && "
" cp -r ${BUILD}/ccnx-src/bin ${SOURCES}"
" )"
)
self.env['PATH'] = "$PATH:${SOURCES}/bin"
def launch(self, check_proto):
peer = self.peer()
local = self.local()
if not peer or not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
peer_port = peer.tun_port
peer_addr = peer.tun_addr
peer_proto = peer.tun_proto
peer_cipher = peer.tun_cipher
local_port = self.port
local_cap = local.capture
self._address = local_addr = local.address
self._netprefix = local_mask = local.netprefix
local_snat = local.snat
local_txq = local.txqueuelen
self._pointopoint = local_p2p = local.pointopoint
local_cipher=local.tun_cipher
local_mcast= local.multicast
local_bwlim= local.bwlimit
local_mcastfwd = local.multicast_forwarder
if not local_p2p and hasattr(peer, 'address'):
self._pointopoint = local_p2p = peer.address
if check_proto != peer_proto:
raise RuntimeError, "Peering protocol mismatch: %s != %s" % (check_proto, peer_proto)
if local_cipher != peer_cipher:
raise RuntimeError, "Peering cipher mismatch: %s != %s" % (local_cipher, peer_cipher)
if check_proto == 'gre' and local_cipher.lower() != 'plain':
raise RuntimeError, "Misconfigured TUN: %s - GRE tunnels do not support encryption. Got %s, you MUST use PLAIN" % (local, local_cipher,)
if local.filter_module:
if check_proto not in ('udp', 'tcp'):
raise RuntimeError, "Miscofnigured TUN: %s - filtered tunnels only work with udp or tcp links" % (local,)
filter_module = filter(bool,map(str.strip,local.filter_module.module.split()))
filter_module = os.path.join('.',os.path.basename(filter_module[0]))
if filter_module.endswith('.c'):
filter_module = filter_module.rsplit('.',1)[0] + '.so'
filter_args = local.filter_module.args
else:
filter_module = None
filter_args = None
args = ["python", "tun_connect.py",
"-m", str(self.mode),
"-t", str(check_proto),
"-A", str(local_addr),
"-M", str(local_mask),
"-C", str(local_cipher),
]
if check_proto == 'fd':
passfd_arg = str(peer_addr)
if passfd_arg.startswith('\x00'):
# cannot shell_encode null characters :(
passfd_arg = "base64:"+base64.b64encode(passfd_arg)
else:
passfd_arg = '$HOME/'+server.shell_escape(passfd_arg)
args.extend([
"--pass-fd", passfd_arg
])
elif check_proto == 'gre':
if self.cross_slice:
args.extend([
"-K", str(self.key.strip('='))
])
args.extend([
"-a", str(peer_addr),
])
# both udp and tcp
else:
args.extend([
"-P", str(local_port),
"-p", str(peer_port),
"-a", str(peer_addr),
"-k", str(self.key)
])
if local_snat:
args.append("-S")
if local_p2p:
args.extend(("-Z",str(local_p2p)))
if local_txq:
args.extend(("-Q",str(local_txq)))
if not local_cap:
args.append("-N")
elif local_cap == 'pcap':
args.extend(('-c','pcap'))
if local_bwlim:
args.extend(("-b",str(local_bwlim*1024)))
if filter_module:
args.extend(("--filter", filter_module))
if filter_args:
args.extend(("--filter-args", filter_args))
if local_mcast and local_mcastfwd:
args.extend(("--multicast-forwarder", local_mcastfwd))
self._logger.info("Starting %s", self)
self._make_home()
self._install_scripts()
# Start process in a "daemonized" way, using nohup and heavy
# stdin/out redirection to avoid connection issues
(out,err),proc = rspawn.remote_spawn(
" ".join(args),
pidfile = './pid',
home = self.home_path,
stdin = '/dev/null',
stdout = 'capture',
stderr = rspawn.STDOUT,
sudo = True,
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key
)
if proc.wait():
raise RuntimeError, "Failed to set up TUN: %s %s" % (out,err,)
self._started = True
def _install_scripts(self):
local = self.local()
if not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
if not local.node:
raise RuntimeError, "Unconnected TUN - missing node"
# Install the tun_connect script and tunalloc utility
from nepi.util import tunchannel
from nepi.util import ipaddr2
sources = [
os.path.join(os.path.dirname(__file__), 'scripts', 'tun_connect.py'),
os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'),
re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__, 1), # pyc/o files are version-specific
re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__, 1), # pyc/o files are version-specific
]
if local.filter_module:
filter_sources = filter(bool,map(str.strip,local.filter_module.module.split()))
filter_module = filter_sources[0]
# Translate paths to builtin sources
for i,source in enumerate(filter_sources):
if not os.path.exists(source):
# Um... try the builtin folder
source = os.path.join(os.path.dirname(__file__), "scripts", source)
if os.path.exists(source):
# Yep... replace
filter_sources[i] = source
sources.extend(set(filter_sources))
else:
filter_module = None
filter_sources = None
dest = "%s@%s:%s" % (
local.node.slicename, local.node.hostname,
os.path.join(self.home_path,'.'),)
(out,err),proc = server.eintr_retry(server.popen_scp)(
sources,
dest,
ident_key = local.node.ident_path,
server_key = local.node.server_key
)
if proc.wait():
raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (sources, out,err,)
# Make sure all dependencies are satisfied
local.node.wait_dependencies()
cmd = ( (
"cd %(home)s && "
"gcc -fPIC -shared tunalloc.c -o tunalloc.so && "
"wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && "
"mkdir -p python-iovec && "
"cd python-iovec && "
"tar xzf ../python-iovec-src.tar.gz --strip-components=1 && "
"python setup.py build && "
"python setup.py install --install-lib .. && "
"cd .. "
+ ( " && "
"gcc -fPIC -shared %(sources)s -o %(module)s.so " % {
'module' : os.path.basename(filter_module).rsplit('.',1)[0],
'sources' : ' '.join(map(os.path.basename,filter_sources))
}
if filter_module is not None and filter_module.endswith('.c')
else ""
)
+ ( " && "
"wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && "
"mkdir -p python-passfd && "
"cd python-passfd && "
"tar xzf ../python-passfd-src.tar.gz --strip-components=1 && "
"python setup.py build && "
"python setup.py install --install-lib .. "
if local.tun_proto == "fd"
else ""
)
)
% {
'home' : server.shell_escape(self.home_path),
'passfd_url' : "http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz",
'iovec_url' : "http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz",
} )
(out,err),proc = server.popen_ssh_command(
cmd,
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 300
)
if proc.wait():
raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (out,err,)
def launch(self, check_proto):
peer = self.peer()
local = self.local()
if not peer or not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
peer_port = peer.tun_port
peer_addr = peer.tun_addr
peer_proto = peer.tun_proto
peer_cipher = peer.tun_cipher
local_port = self.port
local_cap = local.capture
self._address = local_addr = local.address
self._netprefix = local_mask = local.netprefix
local_snat = local.snat
local_txq = local.txqueuelen
self._pointopoint = local_p2p = local.pointopoint
local_cipher = local.tun_cipher
local_mcast = local.multicast
local_bwlim = local.bwlimit
local_mcastfwd = local.multicast_forwarder
if not local_p2p and hasattr(peer, 'address'):
self._pointopoint = local_p2p = peer.address
if check_proto != peer_proto:
raise RuntimeError, "Peering protocol mismatch: %s != %s" % (
check_proto, peer_proto)
if local_cipher != peer_cipher:
raise RuntimeError, "Peering cipher mismatch: %s != %s" % (
local_cipher, peer_cipher)
if check_proto == 'gre' and local_cipher.lower() != 'plain':
raise RuntimeError, "Misconfigured TUN: %s - GRE tunnels do not support encryption. Got %s, you MUST use PLAIN" % (
local,
local_cipher,
)
if local.filter_module:
if check_proto not in ('udp', 'tcp'):
raise RuntimeError, "Miscofnigured TUN: %s - filtered tunnels only work with udp or tcp links" % (
local, )
filter_module = filter(
bool, map(str.strip, local.filter_module.module.split()))
filter_module = os.path.join('.',
os.path.basename(filter_module[0]))
if filter_module.endswith('.c'):
filter_module = filter_module.rsplit('.', 1)[0] + '.so'
filter_args = local.filter_module.args
else:
filter_module = None
filter_args = None
args = [
"python",
"tun_connect.py",
"-m",
str(self.mode),
"-t",
str(check_proto),
"-A",
str(local_addr),
"-M",
str(local_mask),
"-C",
str(local_cipher),
]
if check_proto == 'fd':
passfd_arg = str(peer_addr)
if passfd_arg.startswith('\x00'):
# cannot shell_encode null characters :(
passfd_arg = "base64:" + base64.b64encode(passfd_arg)
else:
passfd_arg = '$HOME/' + server.shell_escape(passfd_arg)
args.extend(["--pass-fd", passfd_arg])
elif check_proto == 'gre':
if self.cross_slice:
args.extend(["-K", str(self.key.strip('='))])
args.extend([
"-a",
str(peer_addr),
])
# both udp and tcp
else:
args.extend([
"-P",
str(local_port), "-p",
str(peer_port), "-a",
str(peer_addr), "-k",
str(self.key)
])
if local_snat:
args.append("-S")
if local_p2p:
args.extend(("-Z", str(local_p2p)))
if local_txq:
args.extend(("-Q", str(local_txq)))
if not local_cap:
args.append("-N")
elif local_cap == 'pcap':
args.extend(('-c', 'pcap'))
if local_bwlim:
args.extend(("-b", str(local_bwlim * 1024)))
if filter_module:
args.extend(("--filter", filter_module))
if filter_args:
args.extend(("--filter-args", filter_args))
if local_mcast and local_mcastfwd:
args.extend(("--multicast-forwarder", local_mcastfwd))
self._logger.info("Starting %s", self)
self._make_home()
self._install_scripts()
# Start process in a "daemonized" way, using nohup and heavy
# stdin/out redirection to avoid connection issues
(out,
err), proc = rspawn.remote_spawn(" ".join(args),
pidfile='./pid',
home=self.home_path,
stdin='/dev/null',
stdout='capture',
stderr=rspawn.STDOUT,
sudo=True,
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key)
if proc.wait():
raise RuntimeError, "Failed to set up TUN: %s %s" % (
out,
err,
)
self._started = True
def __init__(self, api = None):
super(NS3Dependency, self).__init__(api)
self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip bzr'
# We have to download the sources, untar, build...
#pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
pygccxml_source_url = "http://yans.pl.sophia.inria.fr/libs/pygccxml-1.0.0.zip"
ns3_source_url = "http://nepi.inria.fr/code/nepi-ns3.13/archive/tip.tar.gz"
passfd_source_url = "http://nepi.inria.fr/code/python-passfd/archive/tip.tar.gz"
pybindgen_version = "797"
self.build =(
" ( "
" cd .. && "
" python -c 'import pygccxml, pybindgen, passfd' && "
" test -f lib/ns/_core.so && "
" test -f lib/ns/__init__.py && "
" test -f lib/ns/core.py && "
" test -f lib/libns3-core.so && "
" LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
" ) || ( "
# Not working, rebuild
# Archive SHA1 sums to check
"echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
" ( " # check existing files
" sha1sum -c archive_sums.txt && "
" test -f passfd-src.tar.gz && "
" test -f ns3-src.tar.gz "
" ) || ( " # nope? re-download
" rm -rf pybindgen pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
" bzr checkout lp:pybindgen -r %(pybindgen_version)s && " # continue, to exploit the case when it has already been dl'ed
" wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
" wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
" wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
" sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
" ) && "
"unzip -n pygccxml-1.0.0.zip && "
"mkdir -p ns3-src && "
"mkdir -p passfd-src && "
"tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
"tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
"rm -rf target && " # mv doesn't like unclean targets
"mkdir -p target && "
"cd pygccxml-1.0.0 && "
"rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
"python setup.py build && "
"python setup.py install --install-lib ${BUILD}/target && "
"python setup.py clean && "
"cd ../pybindgen && "
"export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
"./waf configure --prefix=${BUILD}/target -d release && "
"./waf && "
"./waf install && "
"./waf clean && "
"mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
"rm -rf ${BUILD}/target/lib && "
"cd ../passfd-src && "
"python setup.py build && "
"python setup.py install --install-lib ${BUILD}/target && "
"python setup.py clean && "
"cd ../ns3-src && "
"./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
"./waf &&"
"./waf install && "
"rm -f ${BUILD}/target/lib/*.so && "
"cp -a ${BUILD}/ns3-src/build/libns3*.so ${BUILD}/target/lib && "
"cp -a ${BUILD}/ns3-src/build/bindings/python/ns ${BUILD}/target/lib &&"
"./waf clean "
" )"
% dict(
pybindgen_version = server.shell_escape(pybindgen_version),
pygccxml_source_url = server.shell_escape(pygccxml_source_url),
ns3_source_url = server.shell_escape(ns3_source_url),
passfd_source_url = server.shell_escape(passfd_source_url),
))
# Just move ${BUILD}/target
self.install = (
" ( "
" cd .. && "
" python -c 'import pygccxml, pybindgen, passfd' && "
" test -f lib/ns/_core.so && "
" test -f lib/ns/__init__.py && "
" test -f lib/ns/core.py && "
" test -f lib/libns3-core.so && "
" LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
" ) || ( "
# Not working, reinstall
"test -d ${BUILD}/target && "
"[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
"( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
"mv -f ${BUILD}/target/* ${SOURCES}"
" )"
)
# Set extra environment paths
self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
def _install_scripts(self):
local = self.local()
if not local:
raise RuntimeError, "Lost reference to peering interfaces before launching"
if not local.node:
raise RuntimeError, "Unconnected TUN - missing node"
# Install the tun_connect script and tunalloc utility
from nepi.util import tunchannel
from nepi.util import ipaddr2
sources = [
os.path.join(os.path.dirname(__file__), 'scripts',
'tun_connect.py'),
os.path.join(os.path.dirname(__file__), 'scripts', 'tunalloc.c'),
re.sub(r"([.]py)[co]$", r'\1', tunchannel.__file__,
1), # pyc/o files are version-specific
re.sub(r"([.]py)[co]$", r'\1', ipaddr2.__file__,
1), # pyc/o files are version-specific
]
if local.filter_module:
filter_sources = filter(
bool, map(str.strip, local.filter_module.module.split()))
filter_module = filter_sources[0]
# Translate paths to builtin sources
for i, source in enumerate(filter_sources):
if not os.path.exists(source):
# Um... try the builtin folder
source = os.path.join(os.path.dirname(__file__), "scripts",
source)
if os.path.exists(source):
# Yep... replace
filter_sources[i] = source
sources.extend(set(filter_sources))
else:
filter_module = None
filter_sources = None
dest = "%s@%s:%s" % (
local.node.slicename,
local.node.hostname,
os.path.join(self.home_path, '.'),
)
(out, err), proc = server.eintr_retry(server.popen_scp)(
sources,
dest,
ident_key=local.node.ident_path,
server_key=local.node.server_key)
if proc.wait():
raise RuntimeError, "Failed upload TUN connect script %r: %s %s" % (
sources,
out,
err,
)
# Make sure all dependencies are satisfied
local.node.wait_dependencies()
cmd = (
("cd %(home)s && "
"gcc -fPIC -shared tunalloc.c -o tunalloc.so && "
"wget -q -c -O python-iovec-src.tar.gz %(iovec_url)s && "
"mkdir -p python-iovec && "
"cd python-iovec && "
"tar xzf ../python-iovec-src.tar.gz --strip-components=1 && "
"python setup.py build && "
"python setup.py install --install-lib .. && "
"cd .. " +
(" && "
"gcc -fPIC -shared %(sources)s -o %(module)s.so " % {
'module': os.path.basename(filter_module).rsplit('.', 1)[0],
'sources': ' '.join(map(os.path.basename, filter_sources))
} if filter_module is not None and filter_module.endswith('.c')
else "") +
(" && "
"wget -q -c -O python-passfd-src.tar.gz %(passfd_url)s && "
"mkdir -p python-passfd && "
"cd python-passfd && "
"tar xzf ../python-passfd-src.tar.gz --strip-components=1 && "
"python setup.py build && "
"python setup.py install --install-lib .. "
if local.tun_proto == "fd" else "")) %
{
'home':
server.shell_escape(self.home_path),
'passfd_url':
"http://nepi.pl.sophia.inria.fr/code/python-passfd/archive/tip.tar.gz",
'iovec_url':
"http://nepi.pl.sophia.inria.fr/code/python-iovec/archive/tip.tar.gz",
})
(out, err), proc = server.popen_ssh_command(
cmd,
host=local.node.hostname,
port=None,
user=local.node.slicename,
agent=None,
ident_key=local.node.ident_path,
server_key=local.node.server_key,
timeout=300)
if proc.wait():
raise RuntimeError, "Failed to set up TUN forwarder: %s %s" % (
out,
err,
)
buildscript = cStringIO.StringIO()
buildscript.write("(\n")
if self.buildDepends:
# Install build dependencies
buildscript.write("sudo -S yum -y install %(packages)s\n" %
{'packages': self.buildDepends})
if self.build:
# Build sources
buildscript.write(
"mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
'command': self._replace_paths(self.build),
'home': server.shell_escape(self.home_path),
})
# Make archive
buildscript.write("tar czf build.tar.gz build\n")
# Write token
buildscript.write(
"echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode"
% {'master_token': server.shell_escape(self._master_token)})
buildscript.seek(0)
return buildscript
def _do_install(self):
def _do_build_slave(self):
if not self.sources and not self.build:
return None
# Create build script
files = set()
if self.sources:
sources = self.sources.split(' ')
files.update(
"%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
os.path.join(self._master.home_path, os.path.basename(source)),)
for source in sources
)
if self.build:
files.add(
"%s@%s:%s" % (self._master.node.slicename, self._master.node.hostip,
os.path.join(self._master.home_path, 'build.tar.gz'),)
)
sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
" && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
" && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
{
'prk' : server.shell_escape(self._master_prk_name),
'puk' : server.shell_escape(self._master_puk_name),
}
kill_agent = "kill $SSH_AGENT_PID"
waitmaster = (
"{ "
"echo 'Checking master reachability' ; "
"if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
"echo 'Master node reachable' ; "
"else "
"echo 'MASTER NODE UNREACHABLE' && "
"exit 1 ; "
"fi ; "
". ./.ssh-agent.sh ; "
"while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
"if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
"}"
) % {
'hostkey' : 'master_known_hosts',
'master' : "%s@%s" % (self._master.node.slicename, self._master.node.hostip),
'master_host' : self._master.node.hostip,
'token_path' : os.path.join(self._master.home_path, 'build.token'),
'token' : server.shell_escape(self._master._master_token),
'sshopts' : sshopts,
}
syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
'hostkey' : 'master_known_hosts',
'files' : ' '.join(files),
'sshopts' : sshopts,
}
if self.build:
syncfiles += " && tar xzf build.tar.gz"
syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(self._master_token),)
syncfiles += " && ( echo %s > build.token.retcode )" % (server.shell_escape(self._master_token),)
syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles,)
cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
'prk' : server.shell_escape(self._master_prk_name),
'puk' : server.shell_escape(self._master_puk_name),
}
slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
'waitmaster' : waitmaster,
'syncfiles' : syncfiles,
'cleanup' : cleanup,
'kill_agent' : kill_agent,
'launch_agent' : launch_agent,
'home' : server.shell_escape(self.home_path),
'token' : server.shell_escape(self._master_token),
}
return cStringIO.StringIO(slavescript)
def _do_build_slave(self):
if not self.sources and not self.build:
return None
# Create build script
files = set()
if self.sources:
sources = self.sources.split(' ')
files.update("%s@%s:%s" % (
self._master.node.slicename,
self._master.node.hostip,
os.path.join(self._master.home_path, os.path.basename(source)),
) for source in sources)
if self.build:
files.add("%s@%s:%s" % (
self._master.node.slicename,
self._master.node.hostip,
os.path.join(self._master.home_path, 'build.tar.gz'),
))
sshopts = "-o ConnectTimeout=30 -o ConnectionAttempts=3 -o ServerAliveInterval=30 -o TCPKeepAlive=yes"
launch_agent = "{ ( echo -e '#!/bin/sh\\ncat' > .ssh-askpass ) && chmod u+x .ssh-askpass"\
" && export SSH_ASKPASS=$(pwd)/.ssh-askpass "\
" && ssh-agent > .ssh-agent.sh ; } && . ./.ssh-agent.sh && ( echo $NEPI_MASTER_PASSPHRASE | ssh-add %(prk)s ) && rm -rf %(prk)s %(puk)s" % \
{
'prk' : server.shell_escape(self._master_prk_name),
'puk' : server.shell_escape(self._master_puk_name),
}
kill_agent = "kill $SSH_AGENT_PID"
waitmaster = (
"{ "
"echo 'Checking master reachability' ; "
"if ping -c 3 %(master_host)s && (. ./.ssh-agent.sh > /dev/null ; ssh -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s echo MASTER SAYS HI ) ; then "
"echo 'Master node reachable' ; "
"else "
"echo 'MASTER NODE UNREACHABLE' && "
"exit 1 ; "
"fi ; "
". ./.ssh-agent.sh ; "
"while [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s.retcode || /bin/true) != %(token)s ]] ; do sleep 5 ; done ; "
"if [[ $(. ./.ssh-agent.sh > /dev/null ; ssh -q -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(master)s cat %(token_path)s || /bin/true) != %(token)s ]] ; then echo BAD TOKEN ; exit 1 ; fi ; "
"}") % {
'hostkey':
'master_known_hosts',
'master':
"%s@%s" %
(self._master.node.slicename, self._master.node.hostip),
'master_host':
self._master.node.hostip,
'token_path':
os.path.join(self._master.home_path, 'build.token'),
'token':
server.shell_escape(self._master._master_token),
'sshopts':
sshopts,
}
syncfiles = ". ./.ssh-agent.sh && scp -p -o UserKnownHostsFile=%(hostkey)s %(sshopts)s %(files)s ." % {
'hostkey': 'master_known_hosts',
'files': ' '.join(files),
'sshopts': sshopts,
}
if self.build:
syncfiles += " && tar xzf build.tar.gz"
syncfiles += " && ( echo %s > build.token )" % (server.shell_escape(
self._master_token), )
syncfiles += " && ( echo %s > build.token.retcode )" % (
server.shell_escape(self._master_token), )
syncfiles = "{ . ./.ssh-agent.sh ; %s ; }" % (syncfiles, )
cleanup = "{ . ./.ssh-agent.sh ; kill $SSH_AGENT_PID ; rm -rf %(prk)s %(puk)s master_known_hosts .ssh-askpass ; }" % {
'prk': server.shell_escape(self._master_prk_name),
'puk': server.shell_escape(self._master_puk_name),
}
slavescript = "( ( %(launch_agent)s && %(waitmaster)s && %(syncfiles)s && %(kill_agent)s && %(cleanup)s ) || %(cleanup)s ) ; echo %(token)s > build.token.retcode" % {
'waitmaster': waitmaster,
'syncfiles': syncfiles,
'cleanup': cleanup,
'kill_agent': kill_agent,
'launch_agent': launch_agent,
'home': server.shell_escape(self.home_path),
'token': server.shell_escape(self._master_token),
}
return cStringIO.StringIO(slavescript)
def wait(self):
local = self.local()
# Wait for the connection to be established
retrytime = 2.0
for spin in xrange(30):
if self.status() != rspawn.RUNNING:
self._logger.warn("FAILED TO CONNECT! %s", self)
break
# Connected?
(out,err),proc = server.eintr_retry(server.popen_ssh_command)(
"cd %(home)s ; grep -a -c Connected capture" % dict(
home = server.shell_escape(self.home_path)),
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 60,
err_on_timeout = False
)
proc.wait()
if out.strip() == '1':
break
# At least listening?
(out,err),proc = server.eintr_retry(server.popen_ssh_command)(
"cd %(home)s ; grep -a -c Listening capture" % dict(
home = server.shell_escape(self.home_path)),
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 60,
err_on_timeout = False
)
proc.wait()
time.sleep(min(30.0, retrytime))
retrytime *= 1.1
else:
(out,err),proc = server.eintr_retry(server.popen_ssh_command)(
"cat %(home)s/capture" % dict(
home = server.shell_escape(self.home_path)),
host = local.node.hostname,
port = None,
user = local.node.slicename,
agent = None,
ident_key = local.node.ident_path,
server_key = local.node.server_key,
timeout = 60,
retry = 3,
err_on_timeout = False
)
proc.wait()
raise RuntimeError, "FAILED TO CONNECT %s: %s%s" % (self,out,err)
if self.buildDepends:
# Install build dependencies
buildscript.write(
"sudo -S yum -y install %(packages)s\n" % {
'packages' : self.buildDepends
}
)
if self.build:
# Build sources
buildscript.write(
"mkdir -p build && ( cd build && ( %(command)s ) )\n" % {
'command' : self._replace_paths(self.build),
'home' : server.shell_escape(self.home_path),
}
)
# Make archive
buildscript.write("tar czf build.tar.gz build\n")
# Write token
buildscript.write("echo %(master_token)s > build.token ) ; echo %(master_token)s > build.token.retcode" % {
'master_token' : server.shell_escape(self._master_token)
})
buildscript.seek(0)
return buildscript
def remote_spawn(command, pidfile, stdout='/dev/null', stderr=STDOUT, stdin='/dev/null', home=None, create_home=False, sudo=False,
host = None, port = None, user = None, agent = None,
ident_key = None, server_key = None,
tty = False, hostip = None):
"""
Spawn a remote command such that it will continue working asynchronously.
Parameters:
command: the command to run - it should be a single line.
pidfile: path of a (ideally unique to this task) pidfile for tracking the process.
stdout: path of a file to redirect standard output to - must be a string.
Defaults to /dev/null
stderr: path of a file to redirect standard error to - string or the special STDOUT value
to redirect to the same file stdout was redirected to. Defaults to STDOUT.
stdin: path of a file with input to be piped into the command's standard input
home: path of a folder to use as working directory - should exist, unless you specify create_home
create_home: if True, the home folder will be created first with mkdir -p
sudo: whether the command needs to be executed as root
host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
Returns:
(stdout, stderr), process
Of the spawning process, which only captures errors at spawning time.
Usually only useful for diagnostics.
"""
# Start process in a "daemonized" way, using nohup and heavy
# stdin/out redirection to avoid connection issues
if stderr is STDOUT:
stderr = '&1'
else:
stderr = ' ' + stderr
daemon_command = '{ { %(command)s > %(stdout)s 2>%(stderr)s < %(stdin)s & } ; echo $! 1 > %(pidfile)s ; }' % {
'command' : command,
'pidfile' : server.shell_escape(pidfile),
'stdout' : stdout,
'stderr' : stderr,
'stdin' : stdin,
}
cmd = "%(create)s%(gohome)s rm -f %(pidfile)s ; %(sudo)s nohup bash -c %(command)s " % {
'command' : server.shell_escape(daemon_command),
'sudo' : 'sudo -S' if sudo else '',
'pidfile' : server.shell_escape(pidfile),
'gohome' : 'cd %s ; ' % (server.shell_escape(home),) if home else '',
'create' : 'mkdir -p %s ; ' % (server.shell_escape,) if create_home else '',
}
(out,err),proc = server.popen_ssh_command(
cmd,
host = host,
port = port,
user = user,
agent = agent,
ident_key = ident_key,
server_key = server_key,
tty = tty ,
hostip = hostip
)
if proc.wait():
raise RuntimeError, "Failed to set up application on host %s: %s %s" % (host, out,err,)
return (out,err),proc
def remote_spawn(command,
pidfile,
stdout='/dev/null',
stderr=STDOUT,
stdin='/dev/null',
home=None,
create_home=False,
sudo=False,
host=None,
port=None,
user=None,
agent=None,
ident_key=None,
server_key=None,
tty=False,
hostip=None):
"""
Spawn a remote command such that it will continue working asynchronously.
Parameters:
command: the command to run - it should be a single line.
pidfile: path of a (ideally unique to this task) pidfile for tracking the process.
stdout: path of a file to redirect standard output to - must be a string.
Defaults to /dev/null
stderr: path of a file to redirect standard error to - string or the special STDOUT value
to redirect to the same file stdout was redirected to. Defaults to STDOUT.
stdin: path of a file with input to be piped into the command's standard input
home: path of a folder to use as working directory - should exist, unless you specify create_home
create_home: if True, the home folder will be created first with mkdir -p
sudo: whether the command needs to be executed as root
host/port/user/agent/ident_key: see nepi.util.server.popen_ssh_command
Returns:
(stdout, stderr), process
Of the spawning process, which only captures errors at spawning time.
Usually only useful for diagnostics.
"""
# Start process in a "daemonized" way, using nohup and heavy
# stdin/out redirection to avoid connection issues
if stderr is STDOUT:
stderr = '&1'
else:
stderr = ' ' + stderr
daemon_command = '{ { %(command)s > %(stdout)s 2>%(stderr)s < %(stdin)s & } ; echo $! 1 > %(pidfile)s ; }' % {
'command': command,
'pidfile': server.shell_escape(pidfile),
'stdout': stdout,
'stderr': stderr,
'stdin': stdin,
}
cmd = "%(create)s%(gohome)s rm -f %(pidfile)s ; %(sudo)s nohup bash -c %(command)s " % {
'command':
server.shell_escape(daemon_command),
'sudo':
'sudo -S' if sudo else '',
'pidfile':
server.shell_escape(pidfile),
'gohome':
'cd %s ; ' % (server.shell_escape(home), ) if home else '',
'create':
'mkdir -p %s ; ' % (server.shell_escape, ) if create_home else '',
}
(out, err), proc = server.popen_ssh_command(cmd,
host=host,
port=port,
user=user,
agent=agent,
ident_key=ident_key,
server_key=server_key,
tty=tty,
hostip=hostip)
if proc.wait():
raise RuntimeError, "Failed to set up application on host %s: %s %s" % (
host,
out,
err,
)
return (out, err), proc
def __init__(self, api=None):
super(NS3Dependency, self).__init__(api)
self.buildDepends = 'make waf gcc gcc-c++ gccxml unzip bzr'
# We have to download the sources, untar, build...
#pygccxml_source_url = "http://leaseweb.dl.sourceforge.net/project/pygccxml/pygccxml/pygccxml-1.0/pygccxml-1.0.0.zip"
pygccxml_source_url = "http://yans.pl.sophia.inria.fr/libs/pygccxml-1.0.0.zip"
ns3_source_url = "http://nepi.inria.fr/code/nepi-ns3.13/archive/tip.tar.gz"
passfd_source_url = "http://nepi.inria.fr/code/python-passfd/archive/tip.tar.gz"
pybindgen_version = "797"
self.build = (
" ( "
" cd .. && "
" python -c 'import pygccxml, pybindgen, passfd' && "
" test -f lib/ns/_core.so && "
" test -f lib/ns/__init__.py && "
" test -f lib/ns/core.py && "
" test -f lib/libns3-core.so && "
" LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
" ) || ( "
# Not working, rebuild
# Archive SHA1 sums to check
"echo '7158877faff2254e6c094bf18e6b4283cac19137 pygccxml-1.0.0.zip' > archive_sums.txt && "
" ( " # check existing files
" sha1sum -c archive_sums.txt && "
" test -f passfd-src.tar.gz && "
" test -f ns3-src.tar.gz "
" ) || ( " # nope? re-download
" rm -rf pybindgen pygccxml-1.0.0.zip passfd-src.tar.gz ns3-src.tar.gz && "
" bzr checkout lp:pybindgen -r %(pybindgen_version)s && " # continue, to exploit the case when it has already been dl'ed
" wget -q -c -O pygccxml-1.0.0.zip %(pygccxml_source_url)s && "
" wget -q -c -O passfd-src.tar.gz %(passfd_source_url)s && "
" wget -q -c -O ns3-src.tar.gz %(ns3_source_url)s && "
" sha1sum -c archive_sums.txt " # Check SHA1 sums when applicable
" ) && "
"unzip -n pygccxml-1.0.0.zip && "
"mkdir -p ns3-src && "
"mkdir -p passfd-src && "
"tar xzf ns3-src.tar.gz --strip-components=1 -C ns3-src && "
"tar xzf passfd-src.tar.gz --strip-components=1 -C passfd-src && "
"rm -rf target && " # mv doesn't like unclean targets
"mkdir -p target && "
"cd pygccxml-1.0.0 && "
"rm -rf unittests docs && " # pygccxml has ~100M of unit tests - excessive - docs aren't needed either
"python setup.py build && "
"python setup.py install --install-lib ${BUILD}/target && "
"python setup.py clean && "
"cd ../pybindgen && "
"export PYTHONPATH=$PYTHONPATH:${BUILD}/target && "
"./waf configure --prefix=${BUILD}/target -d release && "
"./waf && "
"./waf install && "
"./waf clean && "
"mv -f ${BUILD}/target/lib/python*/site-packages/pybindgen ${BUILD}/target/. && "
"rm -rf ${BUILD}/target/lib && "
"cd ../passfd-src && "
"python setup.py build && "
"python setup.py install --install-lib ${BUILD}/target && "
"python setup.py clean && "
"cd ../ns3-src && "
"./waf configure --prefix=${BUILD}/target --with-pybindgen=../pybindgen-src -d release --disable-examples --disable-tests && "
"./waf &&"
"./waf install && "
"rm -f ${BUILD}/target/lib/*.so && "
"cp -a ${BUILD}/ns3-src/build/libns3*.so ${BUILD}/target/lib && "
"cp -a ${BUILD}/ns3-src/build/bindings/python/ns ${BUILD}/target/lib &&"
"./waf clean "
" )" % dict(
pybindgen_version=server.shell_escape(pybindgen_version),
pygccxml_source_url=server.shell_escape(pygccxml_source_url),
ns3_source_url=server.shell_escape(ns3_source_url),
passfd_source_url=server.shell_escape(passfd_source_url),
))
# Just move ${BUILD}/target
self.install = (
" ( "
" cd .. && "
" python -c 'import pygccxml, pybindgen, passfd' && "
" test -f lib/ns/_core.so && "
" test -f lib/ns/__init__.py && "
" test -f lib/ns/core.py && "
" test -f lib/libns3-core.so && "
" LD_LIBRARY_PATH=lib PYTHONPATH=lib python -c 'import ns.core' "
" ) || ( "
# Not working, reinstall
"test -d ${BUILD}/target && "
"[[ \"x\" != \"x$(find ${BUILD}/target -mindepth 1 -print -quit)\" ]] &&"
"( for i in ${BUILD}/target/* ; do rm -rf ${SOURCES}/${i##*/} ; done ) && " # mv doesn't like unclean targets
"mv -f ${BUILD}/target/* ${SOURCES}"
" )")
# Set extra environment paths
self.env['NEPI_NS3BINDINGS'] = "${SOURCES}/lib"
self.env['NEPI_NS3LIBRARY'] = "${SOURCES}/lib"
