def simplify(name, prettyprint=True):
"""
Simplifies all the measure files in pdxml format of the given measure,
converting them using the simplify XSL stylesheet. Old simplifications
will be overwritten.
If the prettyprint optional argument is True, the result will be formatted
using the xmllint tool.
"""
host_shared = settings.PATHS["shared-measures"][0]
pattern = os.path.join(host_shared, name, "*", "*.xml")
simplifier = xml.Transformation(stylesheet("simplify.xsl"))
for source in glob.glob(pattern):
if len(os.path.basename(source).split(".")) == 3:
dest = source.replace(".xml", ".simple.xml")
simplifier.parameters["loopback"] = str(int(source.endswith(".lo.xml")))
simplifier.transform(source, dest)
if prettyprint:
xml.prettyprint(dest)
def report(name, measure_case):
"""
Assembles all the acquired resources (such as source code, measures and
log files) and generates an html page suitable for human interaction and
analysis.
"""
host_shared = settings.PATHS["shared-measures"][0]
trans = xml.Transformation(stylesheet("report.xsl"))
def sources(_):
els = etree.Element("files")
base = len(measure_case) + 1
for root, dirs, files in os.walk(measure_case):
print root
for f in files:
if f.endswith((".pyc", ".DS_Store", ".o")):
continue
path = os.path.join(root, f)
name = path[base:]
if name.startswith("build/"):
continue
element = etree.SubElement(els, "file")
element.attrib["path"] = path
element.attrib["name"] = name
return els
trans.register_function("http://gridgroup.eia-fr.ch/popc", sources)
def logs(_):
els = etree.Element("files")
basel = len(os.path.join(settings.ENV_BASE, host_shared, name))
base = os.path.join(settings.ENV_BASE, host_shared, name, "*.*.*.*", "logs", "*")
for log in glob.glob(base):
element = etree.SubElement(els, "file")
element.attrib["path"] = log
element.attrib["name"] = log[basel + 1 :]
return els
trans.register_function("http://gridgroup.eia-fr.ch/popc", logs)
def format_stream(_, payload):
"""
Stream formatting xslt callback
"""
payload = "".join(payload)
def chunks(seq, n):
""" Yield successive n-sized chunks from l.
"""
for i in xrange(0, len(seq), n):
yield seq[i : i + n]
element = etree.Element("pre")
payload = " ".join(chunks(payload, 2))
payload = " ".join(chunks(payload, 12))
payload = "\n".join(chunks(payload, 104))
for chunk in chunks(payload, 420):
etree.SubElement(element, "span").text = chunk
return element
trans.register_function("http://gridgroup.eia-fr.ch/popc", format_stream)
class Highlighter(etree.XSLTExtension):
def execute(self, context, self_node, input_node, output_parent):
from pygments import highlight
from pygments import lexers
from pygments.formatters import HtmlFormatter
# Highlight source text with pygments
source = input_node.attrib["path"]
with open(source) as fh:
code = fh.read()
# Chose a lexer
name = os.path.split(source)[1]
if name == "Makefile":
lexer = lexers.BaseMakefileLexer()
elif name.endswith(".py"):
lexer = lexers.PythonLexer()
elif name.endswith((".cc", ".ph", ".h")):
lexer = lexers.CppLexer()
elif name.endswith((".c",)):
lexer = lexers.CLexer()
else:
lexer = lexers.TextLexer()
# Highlight code
highlighted = highlight(code, lexer, HtmlFormatter(cssclass="codehilite", style="pastie", linenos="table"))
# Convert to xml
root = etree.fromstring(highlighted)
# Add to parent
output_parent.extend(root)
trans.register_element("http://gridgroup.eia-fr.ch/popc", "highlighted", Highlighter())
destination = os.path.join(host_shared, name, "report")
shutil.rmtree(destination, True)
shell.local("mkdir -p {0}".format(destination))
pattern = os.path.join(host_shared, name, "*", "*.decoded.xml")
for source in glob.glob(pattern):
base, measure = os.path.split(source)
interface = measure.rsplit(".", 3)[1]
ip = os.path.basename(base).replace(".", "-")
dest = os.path.join(destination, "{0}_{1}.html".format(ip, interface))
trans.transform(source, dest)
# Tidy
tconf = "conf/tidy/tidy.conf"
shell.local("tidy -config {1} -o {0} {0} || true".format(dest, tconf))
# Copy resources
htdocs = os.path.join(os.path.dirname(conf.__file__), "htdocs")
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'styles'),
# os.path.join(destination, 'styles')))
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'images'),
# os.path.join(destination, 'images')))
# shell.local("ln -s {0} {1}".format(os.path.join(htdocs, 'scripts'),
# os.path.join(destination, 'scripts')))
shutil.copytree(os.path.join(htdocs, "styles"), os.path.join(destination, "styles"))
shutil.copytree(os.path.join(htdocs, "images"), os.path.join(destination, "images"))
shutil.copytree(os.path.join(htdocs, "scripts"), os.path.join(destination, "scripts"))
def decode(name, measure_case, prettyprint=False):
"""
Decodes the simplified XML representation of the given measure by adding
a "decoded" element to each packet containing a payload.
The decoding is done using an XSL transformation coupled with an xslt
python extension function which provides the "decoded" element given a
payload text string.
"""
host_shared = settings.PATHS["shared-measures"][0]
types = os.path.join(measure_case, "types.py")
types_registry = registry.TypesRegistry()
types_registry.load("pas.conf.basetypes")
try:
types_registry.parse(types)
except IOError:
pass
proto = protocol.MappingProtocol(types_registry)
trans = xml.Transformation(stylesheet("decode.xsl"))
def _decode(context, payload):
"""
Decoding callback
"""
# Convert the ascii representation back to binary data
bin_payload = binascii.a2b_hex("".join(payload))
# Create an xdr stream with the payload
stream = xdrlib.Unpacker(bin_payload)
# Read the full frame length, it is not needed here
_ = stream.unpack_uint()
try:
# Decode the remaining data as a full frame...
# ...hoping that tcp hasn't split the message in more frames
message = proto.decode_full_frame(stream)
# @TODO: Logging, output and error management
except EOFError as e:
print "-" * 80
print context, "Not enough data:", e
print repr(stream.get_buffer())
print "-" * 80
return
except errors.UnknownClass as e:
print "-" * 80
print context.context_node.attrib["timestamp"],
print "Error while decoding packet:", e
print binascii.b2a_hex(stream.get_buffer())
print "-" * 80
return
except errors.UnknownMethod as e:
print "-" * 80
print context.context_node.attrib["timestamp"],
print "Error while decoding packet:", e
print binascii.b2a_hex(stream.get_buffer())
print "-" * 80
return
except xdrlib.Error as e:
print "-" * 80
print context.context_node.attrib["timestamp"], e
print repr(e.message)
rest = stream.get_buffer()
rem = stream.get_position()
print binascii.b2a_hex(rest[rem:])
print
print repr(rest[rem:])
print
print str(rem) + "/" + str(_)
print "*" * 80
return
# Convert the message to xml and send it back to the XSL template
return message.toxml()
trans.register_function("http://gridgroup.eia-fr.ch/popc", _decode, "decode")
# Apply transformation to all simplified xml files
pattern = os.path.join(host_shared, name, "*", "*.simple.xml")
for source in glob.glob(pattern):
dest = source.replace(".simple.xml", ".decoded.xml")
trans.transform(source, dest)
if prettyprint:
xml.prettyprint(dest)
