def consolidateFiles(self, xmlFiles):
"""Given a <files> element, find the directory common to all files
and return a 2-tuple with that directory followed by
a list of files within that directory.
"""
files = []
if xmlFiles:
for fileTag in XML.getChildElements(xmlFiles):
if fileTag.nodeName == 'file':
files.append(XML.shallowText(fileTag))
# If we only have one file, return it as the prefix.
# This prevents the below regex from deleting the filename
# itself, assuming it was a partial filename.
if len(files) == 1:
return files[0], []
# Start with the prefix found by commonprefix,
# then actually make it end with a directory rather than
# possibly ending with part of a filename.
prefix = re.sub("[^/]*$", "", posixpath.commonprefix(files))
endings = []
for file in files:
ending = file[len(prefix):].strip()
if ending == '':
ending = '.'
endings.append(ending)
return prefix, endings
def parse(self, message):
"""Given a string of text in the original CIA commit format, return a <colorText>
element representing it as a DOM tree.
"""
# Initialize our model of the current text format in the original message
self.parsedState = ColorState()
self.document = XML.createRootNode()
# Initialize our stack of (element, ColorState) tuples representing
# the state of the XML document being generated. This starts out with
# our root element in it.
self.elementStack = [
(XML.addElement(self.document, "colorText"), ColorState())
]
# Break up the message into lines, each with its whitespace stripped.
# Run our lexical scanner on each line separately, turning it into
# a stream of events. Insert <br/> tags between lines.
lines = []
for line in message.split("\n"):
# Ignore extra whitespace
line = line.strip()
# Ignore blank lines
if line:
lines.append(line)
for i in xrange(len(lines)):
if i != 0:
XML.addElement(self.elementStack[-1][0], 'br')
self.lex(lines[i])
self.closeTags()
return self.document
def component_files(self, element, args):
"""Format the contents of our <files> tag as a tree with nested lists"""
from cia.LibCIA.Web import Template
files = XML.dig(args.message.xml, "message", "body", "commit", "files")
if not (files and XML.hasChildElements(files)):
return []
# First we organize the files into a tree of nested dictionaries.
# The dictionary we ultimately have FileTree render maps each node
# (file or directory) to a dictionary of its contents. The keys
# in these dictionaries can be any Nouvelle-renderable object
# produced by format_file.
#
# As a first step, we build a dictionary mapping path segment to
# [fileTag, children] lists. We then create a visual representation
# of each fileTag and generate the final dictionary.
fileTree = {}
for fileTag in XML.getChildElements(files):
if fileTag.nodeName == 'file':
# Separate the file into path segments and walk into our tree
node = [None, fileTree]
for segment in XML.shallowText(fileTag).split('/'):
if segment:
node = node[1].setdefault(segment, [None, {}])
# The leaf node owns this fileTag
node[0] = fileTag
return [Template.FileTree(self.format_file_tree(fileTree))]
def preprocess(self):
message = XML.dig(self.xml, "message")
if not message:
raise XML.XMLValidityError("A Message's root node must be named 'message'")
# Stamp it with the current time if it has no timestamp yet
if not XML.dig(message, "timestamp"):
XML.addElement(message, "timestamp", "%d" % time.time())
def dbIter(self):
if os.path.isfile(self.path):
dom = XML.parseStream(open(self.path))
for element in XML.getChildElements(dom.documentElement):
if element.nodeName == "ruleset":
yield element
else:
log.msg("The file %r does not exist, loading no rulesets" % self.path)
def format(self, args):
# Format each package inside each result set
packages = []
for results in XML.getChildElements(XML.dig(args.message.xml, "message", "body", "builder")):
if results.nodeName == 'results':
for package in XML.getChildElements(results):
if package.nodeName == 'package':
packages.append(self.format_package(package))
return self.joinMessage(args.message, packages)
def textComponent(self, element, args, *path):
"""A convenience function for defining components that just look for a node
in the message and return its shallowText.
"""
element = XML.dig(args.message.xml, *path)
if element:
return [XML.shallowText(element)]
else:
return [MarkAsHidden()]
def format(self, args):
if not args.input:
return
project = XML.dig(args.message.xml, "message", "source", "project")
if project:
from cia.LibCIA.IRC.Formatting import format
prefix = format("%s:" % XML.shallowText(project), 'bold') + " "
return "\n".join([prefix + line for line in args.input.split("\n")])
else:
return args.input
def getXMLMailHeaders(self, document):
"""Return a <mailHeaders> tag representing a subset of the headers
for this message. This is placed in the <generator> tag of any
message passing through this module, to document and log the
message's true source.
"""
node = document.createElementNS(None, "mailHeaders")
for name, value in self.message.items():
if name in interestingHeaders:
XML.addElement(node, "header", content=str(value)).setAttributeNS(None, 'name', name)
return node
def format_generator(self, gen):
"""Format the information contained in this message's <generator> tag"""
name = XML.digValue(gen, str, "name")
url = XML.digValue(gen, str, "url")
version = XML.digValue(gen, str, "version")
if url:
name = tag('a', href=url)[ name ]
items = ["Generated by ", Template.value[ name ]]
if version:
items.extend([" version ", version])
return items
def render_rows(self, context):
if not self.message:
return []
rows = []
timestamp = XML.dig(self.message.xml, "message", "timestamp")
if timestamp:
rows.append(self.format_timestamp(timestamp))
generator = XML.dig(self.message.xml, "message", "generator")
if generator:
rows.append(self.format_generator(generator))
return rows
def getSvnRevision(self):
"""Return the current Subversion repository revision, or None
if we're not in an svn working copy or it can't be parsed.
"""
try:
entries = XML.parseString(open(".svn/entries").read()).documentElement
highestRev = 0
for tag in XML.getChildElements(entries):
if tag.nodeName == 'entry':
rev = tag.getAttributeNS(None, 'committed-rev')
if rev and rev > highestRev:
highestRev = rev
return highestRev
except:
return None
def format_results(self, package):
"""Given a package, returns a formatted representation of all results for that package"""
results = []
for element in XML.getChildElements(package):
f = getattr(self, 'result_' + element.nodeName, None)
if f:
results.append(f(element))
def render_message(self, context):
if not self.message:
context['request'].setResponseCode(404)
return self.notFoundMessage
# Try to format it using several media, in order of decreasing preference.
# The 'xhtml-long' formatter lets messages define a special formatter to
# use when an entire page is devoted to their one message, possibly showing
# it in greater detail. 'xhtml' is the formatter most messages should have.
# 'plaintext' is a nice fallback.
#
# This default list of media to try can be overridden with an argument in our URL.
if 'media' in context['args']:
mediaList = context['args']['media'][0].split()
else:
mediaList = ('xhtml-long', 'xhtml', 'plaintext')
for medium in mediaList:
try:
formatted = Formatters.getFactory().findMedium(
medium, self.message).formatMessage(self.message)
except Message.NoFormatterError:
continue
return formatted
# Still no luck? Display a warning message and a pretty-printed XML tree
return [
tag('h1')[ "No formatter available" ],
XML.htmlPrettyPrint(self.message.xml),
]
def syncFromServer(self):
"""Update this Bot from the RPC server, if necessary.
Right now the only task this performs is to store the
server's ruleset if filterMode is 'unknown'.
"""
if self.filter_mode != FILTER.UNKNOWN:
return
ruleset = self._loadRuleset()
if not ruleset:
# If there's no ruleset, mark the bot as inactive.
self.filter_mode = FILTER.INACTIVE
self.save()
return
# Parse the ruleset using LibCIA's XML library
from cia.LibCIA import XML
dom = XML.parseString(ruleset)
# XXX: We should try to reduce the ruleset to one of
# the other FILTER.* modes if possible. For now,
# we'll always import existing rulesets as
# FILTER.CUSTOM.
# Flatten the contents of the <ruleset> element, clean up
# the resulting text, and save that as a custom filter.
text = ''.join([n.toxml() for n in dom.documentElement.childNodes])
self.filter_mode = FILTER.CUSTOM
self.custom_ruleset = clean_up_text(text)
self.save()
def element_like(self, element):
"""Compare a given variable to the element's content using SQL's 'LIKE' operator,
not including leading and trailing whitespace. This is case-insensitive, and includes
the '%' wildcard which may be placed at the beginning or end of the string.
"""
return "(%s LIKE %s)" % (self.varLookup(element.getAttributeNS(None, 'var')),
quote(XML.shallowText(element).strip(), 'varchar'))
def rulesetReturn(msg):
nodes = xp.queryObject(msg)
if nodes:
self.result = XML.allText(nodes[0]).strip()
else:
self.result = None
raise RulesetReturnException()
def param_noColor(self, tag):
"""The <noColor> parameter disables colors.
This is equivalent to a <format> parameter with CommitFormatter's
default component tree.
"""
self.componentTree = XML.parseString(CommitFormatter.defaultComponentTree
).documentElement
def matches(self, msg):
match = self.projectPath.queryObject(msg)
if match:
for node in match:
project = XML.shallowText(node)
if (project.strip().lower() in self.projects):
return True
return False
def caps_store(self, path, xml):
"""Generate a list of acceptable capabilities to grant access to 'store'.
In addition to the usual ones, allow ('ruleset.uri', x) where x is the
ruleset's URI.
"""
if (xml.startswith('[')):
uri = xml.split('\n')[0][2:]
else:
uri = XML.parseString(xml).documentElement.getAttributeNS(None, 'uri')
return self.makeDefaultCaps(path) + [('ruleset.uri', uri)]
def render(self, request):
xml = self.target.messages.getMessageById(self.id)
if xml:
request.setHeader('content-type', 'text/xml')
request.write(unicode(XML.toString(xml)).encode('utf-8'))
request.finish()
else:
request.write(error.NoResource("Message not found").render(request))
request.finish()
return ""
def component_files(self, element, args):
"""Break up our list of files into a common prefix and a sensibly-sized
list of filenames after that prefix.
"""
files = XML.dig(args.message.xml, "message", "body", "commit", "files")
if not (files and XML.hasChildElements(files)):
return [Message.MarkAsHidden()]
prefix, endings = self.consolidateFiles(files)
endingStr = " ".join(endings)
if len(endingStr) > self.filesWidthLimit:
# If the full file list is too long, give a file summary instead
endingStr = self.summarizeFiles(endings)
if prefix.startswith('/'):
prefix = prefix[1:]
if endingStr:
return ["%s (%s)" % (prefix, endingStr)]
else:
return [prefix]
def postprocessMessage(self, xml):
"""Gets a chance to modify all XML messages before they're loaded
and dispatched to the Hub. This does the following:
- If there is no <generator> at all, adds a generic one
- Removes any <mailHeaders> tag that may already exist in <generator>
- Adds a correct <mailHeaders> tag to the <generator>
"""
# Create the <generator> tag if it doesn't exist
if not XML.dig(xml, "message", "generator"):
xml.documentElement.appendChild(self.getLocalGenerator(xml))
generator = XML.dig(xml, "message", "generator")
# Delete an existing <mailHeaders>
for child in list(XML.getChildElements(generator)):
if child.nodeName == "mailHeaders":
generator.removeChild(child)
# Add a new <mailHeaders>
generator.appendChild(self.getXMLMailHeaders(xml))
return xml
def command_Announce(self, project):
"""Old-style announcements: Announce <project> in the subject line.
The body of the email contained the message's text, marked up
with {color} tags but with no metadata.
"""
xml = XML.createRootNode()
# Convert the given project name to a <project> tag inside <source>,
# after filtering it a bit... in the old CIA project names and IRC channel
# names weren't particularly distinct, so preceeding "#" characters on
# projects were ignored. We preserve this behaviour.
if project[0] == "#":
project = project[1:]
XML.buryValue(xml, project, "message", "source", "project")
# Since old-style commits didn't have any metadata, the best we can do
# is to represent the log in a <colorText> element
colorText = ColorTextParser().parse(self.message.get_payload()).documentElement
XML.bury(xml, "message", "body").appendChild(xml.importNode(colorText, True))
return xml
def format(self, args):
"""The formatter entry point. This just finds the current component
tree and invokes walkComponents and joinComponents on it.
"""
# Parse the default component tree, caching it per-class
if self.__class__._defaultTreeOwner is not self.__class__.defaultComponentTree:
self.__class__._defaultTreeOwner = self.__class__.defaultComponentTree
self.__class__._cachedDefaultTree = XML.parseString(self.__class__.defaultComponentTree).documentElement
# This will use the default component tree if it hasn't been overridden in this instance
tree = self.componentTree or self.__class__._cachedDefaultTree
return self.joinComponents(self.walkComponents(tree.childNodes, args))
def joinMessage(self, message, packages):
content = []
branch = XML.digValue(message.xml, str, "message", "source", "branch")
if branch:
content.append(Nouvelle.tag('strong')[ self.format_branch(branch.strip()) ])
for package in packages:
if content:
content.append(Nouvelle.tag('br'))
content.append(package)
return content
def joinMessage(self, message, packages):
"""Join the results for each package into a final result"""
content = "builder"
branch = XML.digValue(message.xml, str, "message", "source", "branch")
if branch:
content += " " + self.format_branch(branch.strip())
# If we have only one package, put it on the same line as the heading
if len(packages) <= 1:
content += ": " + packages[0]
else:
content += "\n" + "\n".join(packages)
return content
def component_text(self, element, args):
"""This is a generic version of textComponent, in which 'path' can
be specified by users. Any textComponent can be rewritten as a
<text> component.
"""
path = element.getAttributeNS(None, 'path')
if not path:
raise XML.XMLValidityError("The 'path' attribute on <text> is required.")
xp = XML.XPath(XML.pathShortcuts.get(path, path))
nodes = xp.queryObject(args.message)
if nodes:
return [XML.shallowText(nodes[0])]
else:
return [MarkAsHidden()]
def pathMatchTag(self, element, function, textExtractor=XML.shallowText):
"""Implements the logic common to all tags that test the text matched by
an XPath against the text inside our element. The given function is used
to determine if the text matches. This implements the properties common to
several elements:
- The caseSensitive attribute defaults to 1, but can be set to zero
to force both strings to lowercase.
- Each XPath match is tested separately, with 'or' semantics:
if any of the XPath matches cause the provided function to match,
this returns True
- If there are no XPath matches, returns False
"""
path = element.getAttributeNS(None, 'path')
xp = XML.XPath(XML.pathShortcuts.get(path, path))
# Are we doing a case sensitive match? Default is no.
caseSensitive = element.getAttributeNS(None, 'caseSensitive')
if caseSensitive:
caseSensitive = int(caseSensitive)
else:
caseSensitive = 0
text = XML.shallowText(element).strip()
if not caseSensitive:
text = text.lower()
def filterMatch(msg):
# Use queryobject then str() so that matched
# nodes without any text still give us at least
# the empty string. This is important so that <find>
# with an empty search string can be used to test
# for the existence of an XPath match.
nodes = xp.queryObject(msg)
if nodes:
matchStrings = map(textExtractor, nodes)
# Any of the XPath matches can make our match true
for matchString in matchStrings:
matchString = matchString.strip()
if not caseSensitive:
matchString = matchString.lower()
if function(matchString, text):
return True
return False
return filterMatch
def extractSummary(element, widthLimit=80):
"""Extract all text from the given XML element, remove extra
whitespace, and truncate it to no longer than the given width.
"""
# Extract all text, eating extra whitespace
text = re.sub("\s+", " ", XML.allText(element)).strip()
# Use wrapLine to cleanly break it if possible, but
# truncate it if necessary- wrapLine will not break words in
# half if they are longer than the wrap width.
lines = wrapLine(text, widthLimit)
if lines:
summary = lines[0][:widthLimit]
if len(summary) < len(text):
summary += "..."
return summary