def command(_=None):
"""
Exchanges the public keys between all VMs to allow direct ssh connections
between them without user input.
"""
local, remote = settings.PATHS['configuration']
local = os.path.join(local, 'authorized_keys')
remote = os.path.join(remote, 'authorized_keys')
with shell.ignore_warnings():
shell.local('rm {0}'.format(local))
# Collect all keys in one file
shell.remote('cat $HOME/.ssh/id_rsa.pub >>{0}'.format(remote))
# Copy first authorized key (host machine) to temp location
shell.remote('head -1 $HOME/.ssh/authorized_keys ' \
'>$HOME/.ssh/authorized_keys.tmp')
# Append all other keys
shell.remote('cat {0} >>$HOME/.ssh/authorized_keys.tmp'.format(remote))
# Move to the original location
shell.remote('mv $HOME/.ssh/authorized_keys.tmp ' \
'$HOME/.ssh/authorized_keys')
# Add all hosts to the known_hosts file
shell.remote('cat $HOME/.ssh/authorized_keys | '\
'awk -F \' \' \'{print $3}\' | ' \
'grep -E \'[0-9.]{7,15}\' | ' \
'ssh-keyscan -f - -H -t rsa >$HOME/.ssh/known_hosts')
shell.local('rm {0}'.format(local))
def compile(name, localdir=None, remotedir=None):
"""
Compiles the given test case on all the hosts in the system.
The building details and the respect of the convention are enforced by the
Makefile and not further explained here.
"""
# Read the defaults from the settings if the arguments are not provided
if not localdir or not remotedir:
paths = settings.PATHS['test-cases']
if not localdir:
localdir = paths[0]
if not remotedir:
remotedir = paths[1]
local = os.path.join(localdir, name)
remote = os.path.join(remotedir, name)
shell.local('rm -f {0}/build/obj.map'.format(local))
base = os.path.dirname(settings.PATHS['configuration'][1])
with shell.workon(all_hosts()):
with shell.cd(remote):
shell.remote('ENV_BASE={0} make -e clean'.format(base))
shell.remote('ENV_BASE={0} make -e build'.format(base))
def collect(name, overwrite=False):
"""
Moves the relevant files to the shared directory by asking to empty the
destination directory if needed.
"""
ipaddr = "$(getip eth1)"
name = "{0}_{1}".format(name, datetime.now().strftime("%Y-%m-%d_%H:%M"))
guest_local = settings.PATHS["local-measures"][1]
host_shared, guest_shared = settings.PATHS["shared-measures"]
destination = os.path.join(guest_shared, name, ipaddr)
local = os.path.realpath(os.path.join(host_shared, name))
try:
if os.listdir(local):
print "A directory with the same name ({0}) already " "exists.".format(name)
if overwrite or shell.confirm("Would you like to replace it?"):
shell.local("rm -rf {0}/*".format(local))
else:
raise OSError(errno.ENOTEMPTY, "Directory not empty")
except OSError as e:
# If the file or directory don't exist, consume the exception
if e.errno != errno.ENOENT:
raise
shell.remote("chown -R {0}:{0} {1}".format(settings.VM_USER, guest_local), sudo=True)
shell.remote('mkdir -p "{0}/logs"'.format(destination))
shell.remote('cp {0}/* "{1}"'.format(guest_local, destination))
# Copy log files
for logfile in settings.LOG_FILES:
shell.remote('chown {0}:{0} "{1}" || true'.format(settings.VM_USER, logfile), sudo=True)
shell.remote('cp "{0}" "{1}/logs" || true'.format(logfile, destination))
def command(options):
"""
Exchanges the public keys between all VMs to allow direct ssh connections
between them without user input.
"""
local, _ = settings.PATHS['test-cases']
shell.local("ENV_BASE='{2}' EXEC={0[1]} make -e -f {0[0]}/Makefile " \
"{1}".format(options.case, " ".join(options.targets), settings.ENV_BASE))
def report(name, measure_case):
"""
Assembles all the acquired resources (such as source code, measures and
log files) and generates an html page suitable for human interaction and
analysis.
"""
host_shared = settings.PATHS["shared-measures"][0]
trans = xml.Transformation(stylesheet("report.xsl"))
def sources(_):
els = etree.Element("files")
base = len(measure_case) + 1
for root, dirs, files in os.walk(measure_case):
print root
for f in files:
if f.endswith((".pyc", ".DS_Store", ".o")):
continue
path = os.path.join(root, f)
name = path[base:]
if name.startswith("build/"):
continue
element = etree.SubElement(els, "file")
element.attrib["path"] = path
element.attrib["name"] = name
return els
trans.register_function("http://gridgroup.eia-fr.ch/popc", sources)
def logs(_):
els = etree.Element("files")
basel = len(os.path.join(settings.ENV_BASE, host_shared, name))
base = os.path.join(settings.ENV_BASE, host_shared, name, "*.*.*.*", "logs", "*")
for log in glob.glob(base):
element = etree.SubElement(els, "file")
element.attrib["path"] = log
element.attrib["name"] = log[basel + 1 :]
return els
trans.register_function("http://gridgroup.eia-fr.ch/popc", logs)
def format_stream(_, payload):
"""
Stream formatting xslt callback
"""
payload = "".join(payload)
def chunks(seq, n):
""" Yield successive n-sized chunks from l.
"""
for i in xrange(0, len(seq), n):
yield seq[i : i + n]
element = etree.Element("pre")
payload = " ".join(chunks(payload, 2))
payload = " ".join(chunks(payload, 12))
payload = "\n".join(chunks(payload, 104))
for chunk in chunks(payload, 420):
etree.SubElement(element, "span").text = chunk
return element
trans.register_function("http://gridgroup.eia-fr.ch/popc", format_stream)
class Highlighter(etree.XSLTExtension):
def execute(self, context, self_node, input_node, output_parent):
from pygments import highlight
from pygments import lexers
from pygments.formatters import HtmlFormatter
# Highlight source text with pygments
source = input_node.attrib["path"]
with open(source) as fh:
code = fh.read()
# Chose a lexer
name = os.path.split(source)[1]
if name == "Makefile":
lexer = lexers.BaseMakefileLexer()
elif name.endswith(".py"):
lexer = lexers.PythonLexer()
elif name.endswith((".cc", ".ph", ".h")):
lexer = lexers.CppLexer()
elif name.endswith((".c",)):
lexer = lexers.CLexer()
else:
lexer = lexers.TextLexer()
# Highlight code
highlighted = highlight(code, lexer, HtmlFormatter(cssclass="codehilite", style="pastie", linenos="table"))
# Convert to xml
root = etree.fromstring(highlighted)
# Add to parent
output_parent.extend(root)
trans.register_element("http://gridgroup.eia-fr.ch/popc", "highlighted", Highlighter())
destination = os.path.join(host_shared, name, "report")
shutil.rmtree(destination, True)
shell.local("mkdir -p {0}".format(destination))
pattern = os.path.join(host_shared, name, "*", "*.decoded.xml")
for source in glob.glob(pattern):
base, measure = os.path.split(source)
interface = measure.rsplit(".", 3)[1]
ip = os.path.basename(base).replace(".", "-")
dest = os.path.join(destination, "{0}_{1}.html".format(ip, interface))
trans.transform(source, dest)
# Tidy
tconf = "conf/tidy/tidy.conf"
shell.local("tidy -config {1} -o {0} {0} || true".format(dest, tconf))
# Copy resources
htdocs = os.path.join(os.path.dirname(conf.__file__), "htdocs")
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'styles'),
# os.path.join(destination, 'styles')))
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'images'),
# os.path.join(destination, 'images')))
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'scripts'),
# os.path.join(destination, 'scripts')))
shutil.copytree(os.path.join(htdocs, "styles"), os.path.join(destination, "styles"))
shutil.copytree(os.path.join(htdocs, "images"), os.path.join(destination, "images"))
shutil.copytree(os.path.join(htdocs, "scripts"), os.path.join(destination, "scripts"))
