def failUnlessXmlError(self,f,args,exc,domain,code,message,level,file,line):
"""Run function f, with arguments args and expect an exception exc;
when the exception is raised, check the libxml2.lastError for
expected values."""
# disable the default error handler
libxml2.registerErrorHandler(None,None)
try:
apply(f,args)
except exc:
e = libxml2.lastError()
if e is None:
self.fail("lastError not set")
if 0:
print "domain = ",e.domain()
print "code = ",e.code()
print "message =",repr(e.message())
print "level =",e.level()
print "file =",e.file()
print "line =",e.line()
print
self.failUnlessEqual(domain,e.domain())
self.failUnlessEqual(code,e.code())
self.failUnlessEqual(message,e.message())
self.failUnlessEqual(level,e.level())
self.failUnlessEqual(file,e.file())
self.failUnlessEqual(line,e.line())
else:
self.fail("exception %s should have been raised" % exc)
def parse_node_helper(xml, root_name, callback, exec_class=ValueError):
"""
Parse the passed XML, expecting root as root_name, and pass the
root node to callback
"""
class ErrorHandler:
def __init__(self):
self.msg = ""
def handler(self, ignore, s):
self.msg += s
error = ErrorHandler()
libxml2.registerErrorHandler(error.handler, None)
try:
try:
doc = libxml2.readMemory(xml, len(xml),
None, None,
libxml2.XML_PARSE_NOBLANKS)
except (libxml2.parserError, libxml2.treeError), e:
raise exec_class("%s\n%s" % (e, error.msg))
finally:
libxml2.registerErrorHandler(None, None)
ret = None
try:
root = doc.getRootElement()
if root.name != root_name:
raise ValueError("Root element is not '%s'" % root_name)
ret = callback(root)
finally:
doc.freeDoc()
return ret
def parse(xml, filename):
"""Parse the XML description of a VM image into a data structure. Returns
an object of class Image. BASE should be the directory where the disk
image files for this image can be found"""
class ErrorHandler:
def __init__(self):
self.msg = ""
def handler(self, ignore, s):
self.msg += s
error = ErrorHandler()
libxml2.registerErrorHandler(error.handler, None)
try:
try:
doc = libxml2.readMemory(xml, len(xml),
None, None,
libxml2.XML_PARSE_NOBLANKS)
except (libxml2.parserError, libxml2.treeError), e:
raise ParserException("%s\n%s" % (e, error.msg))
finally:
libxml2.registerErrorHandler(None, None)
try:
root = doc.getRootElement()
if root.name != "image":
raise ParserException(_("Root element is not 'image'"))
image = Image(root, filename=filename)
finally:
doc.freeDoc()
return image
def failUnlessXmlError(self, f, args, exc, domain, code, message, level,
file, line):
"""Run function f, with arguments args and expect an exception exc;
when the exception is raised, check the libxml2.lastError for
expected values."""
# disable the default error handler
libxml2.registerErrorHandler(None, None)
try:
apply(f, args)
except exc:
e = libxml2.lastError()
if e is None:
self.fail("lastError not set")
if 0:
print "domain = ", e.domain()
print "code = ", e.code()
print "message =", repr(e.message())
print "level =", e.level()
print "file =", e.file()
print "line =", e.line()
print
self.failUnlessEqual(domain, e.domain())
self.failUnlessEqual(code, e.code())
self.failUnlessEqual(message, e.message())
self.failUnlessEqual(level, e.level())
self.failUnlessEqual(file, e.file())
self.failUnlessEqual(line, e.line())
else:
self.fail("exception %s should have been raised" % exc)
def validate_schema(tdl_file):
# Locate relaxng schema
rng_file = None
for tryme in ["../../docs/tdl.rng", "../docs/tdl.rng", "docs/tdl.rng", "tdl.rng"]:
if os.path.isfile(tryme):
rng_file = tryme
break
if rng_file is None:
raise Exception("RelaxNG schema file not found: tdl.rng")
# Load relaxng schema
schema = open(rng_file, "r").read()
rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
rngs = rngp.relaxNGParse()
# Define callback for error handling
def error_cb(ctx, str):
print "%s: %s" % (ctx, str.strip())
libxml2.registerErrorHandler(error_cb, tdl_file)
# Attempt to validate
reader = libxml2.newTextReaderFilename(tdl_file)
# reader.SetParserProp(libxml2.PARSER_VALIDATE, 1)
reader.RelaxNGSetSchema(rngs)
ret = reader.Read()
while ret == 1:
ret = reader.Read()
if ret != 0:
raise Exception("Error parsing the document: %s" % tdl_file)
if reader.IsValid() != 1:
raise Exception("Document failed to validate: %s" % tdl_file)
def parse_node_helper(xml, root_name, callback, exec_class=ValueError):
"""
Parse the passed XML, expecting root as root_name, and pass the
root node to callback
"""
class ErrorHandler:
def __init__(self):
self.msg = ""
def handler(self, ignore, s):
self.msg += s
error = ErrorHandler()
libxml2.registerErrorHandler(error.handler, None)
try:
try:
doc = libxml2.readMemory(xml, len(xml),
None, None,
libxml2.XML_PARSE_NOBLANKS)
except (libxml2.parserError, libxml2.treeError), e:
raise exec_class("%s\n%s" % (e, error.msg))
finally:
libxml2.registerErrorHandler(None, None)
ret = None
try:
root = doc.getRootElement()
if root.name != root_name:
raise ValueError("Root element is not '%s'" % root_name)
ret = callback(root)
finally:
doc.freeDoc()
return ret
def parse(xml, filename):
"""Parse the XML description of a VM image into a data structure. Returns
an object of class Image. BASE should be the directory where the disk
image files for this image can be found"""
class ErrorHandler:
def __init__(self):
self.msg = ""
def handler(self, ignore, s):
self.msg += s
error = ErrorHandler()
libxml2.registerErrorHandler(error.handler, None)
try:
try:
doc = libxml2.readMemory(xml, len(xml), None, None,
libxml2.XML_PARSE_NOBLANKS)
except (libxml2.parserError, libxml2.treeError), e:
raise ParserException("%s\n%s" % (e, error.msg))
finally:
libxml2.registerErrorHandler(None, None)
try:
root = doc.getRootElement()
if root.name != "image":
raise ParserException(_("Root element is not 'image'"))
image = Image(root, filename=filename)
finally:
doc.freeDoc()
return image
def setUp(self):
#
# Set up Libxml2.
#
self.initialMemUsed = libxml2.debugMemory(1)
libxml2.initParser()
libxml2.lineNumbersDefault(1)
libxml2.registerErrorHandler(handleError, self)
def setUp(self): # # Set up Libxml2. # self.initialMemUsed = libxml2.debugMemory(1) libxml2.initParser() libxml2.lineNumbersDefault(1) libxml2.registerErrorHandler(handleError, self)
def transform(doc, styleSheet, params=None):
# Redirect libxml error messages
libxml2.registerErrorHandler(callback, None)
libxslt.registerErrorHandler(callback, None)
# Transforms doc (XML data) with style (XSLT stylesheet file name)
try:
styledoc = libxml2.parseFile(styleSheet)
except libxml2.parserError, e:
raise "Could not parse %s" % styleSheet, e
def xmlToRoadmap(filePath):
libxml2.registerErrorHandler(lambda ctx,str: None, None)
libxslt.registerErrorHandler(lambda ctx,str: None, None)
pmlStyleDoc=libxml2.parseFile(settings.PML_PATH + "/xpml/pmldoc.xsl")
style = libxslt.parseStylesheetDoc(pmlStyleDoc)
try:
doc = libxml2.parseFile(filePath)
result = style.applyStylesheet(doc, None)
return style.saveResultToString(result)
except (libxml2.parserError, TypeError):
return None
def __init__(self, xfile, isDoc=False):
try:
self.doc = None
if isDoc:
self.fname = ''
self.doc = libxml2.parseDoc(xfile)
else:
self.fname = xfile
self.doc = libxml2.parseFile(xfile)
except libxml2.parserError:
raise UniXMLException("(UniXML): Can`t open file " + xfile)
libxml2.registerErrorHandler(self.callback, "-->")
def __init__(self, xfile, isDoc=False):
try:
self.doc = None
if isDoc:
self.fname = ''
self.doc = libxml2.parseDoc(xfile)
else:
self.fname = xfile
self.doc = libxml2.parseFile(xfile)
except libxml2.parserError:
raise UniXMLException("(UniXML): Can`t open file " + xfile)
libxml2.registerErrorHandler(self.callback, "-->")
def __initParseFileXML():
# Use libxml2 if available, it gives us better error checking than
# xml.dom.minidom (DTD validation, line numbers etc.)
global parseFileXML
try:
global libxml2
import libxml2
parseFileXML = __parseFileLibxml2
libxml2.registerErrorHandler(__libxml2err, "-->")
except(ImportError):
parseFileXML = __parseFileMinidom
global xml
import sys, xml.sax, xml.dom, xml.dom.minidom, xml.parsers.expat
def __initParseFileXML():
# Use libxml2 if available, it gives us better error checking than
# xml.dom.minidom (DTD validation, line numbers etc.)
global parseFileXML
try:
global libxml2
import libxml2
parseFileXML = __parseFileLibxml2
libxml2.registerErrorHandler(__libxml2err, "-->")
except(ImportError):
parseFileXML = __parseFileMinidom
global xml
import sys, xml.sax, xml.dom, xml.dom.minidom, xml.parsers.expat
def xmlToPml(filePath):
#Redirect XML errors away from stdout so that it doesn't cause a
#mod_wsgi exception.
libxml2.registerErrorHandler(lambda ctx,str: None, None)
libxslt.registerErrorHandler(lambda ctx,str: None, None)
pmlStyleDoc=libxml2.parseFile(settings.PML_PATH + "/xpml/xpml.xsl")
style = libxslt.parseStylesheetDoc(pmlStyleDoc)
try:
doc = libxml2.parseFile(filePath)
result = style.applyStylesheet(doc, None)
return style.saveResultToString(result)
except (libxml2.parserError, TypeError):
return None
def setUp(self):
def callback(ctx, str):
pass #FIXME! if self.verbose >= 0: print "%s %s" % (ctx, str)
libxml2.registerErrorHandler(callback, "-->")
self.Config = pokerengineconfig.Config([PokerEngineConfigTestCase.TestConfDirectory, tempfile.gettempdir()])
self.ConfigTmplFile = path.join(PokerEngineConfigTestCase.TestConfDirectory, PokerEngineConfigTestCase.TestConfigTemplateFile)
self.ConfigTempFile = path.join(tempfile.gettempdir(), PokerEngineConfigTestCase.TestConfigTemporaryFile)
self.UpgradeTmplFile = path.join(PokerEngineConfigTestCase.TestUpgradeDirectory, PokerEngineConfigTestCase.TestUpgradeTemplateFile)
self.UpgradeInvalidFile = path.join(PokerEngineConfigTestCase.TestUpgradeDirectory, PokerEngineConfigTestCase.TestUpgradeInvalidFile)
if not self.CopyFile(self.ConfigTmplFile, self.ConfigTempFile):
self.fail('Error during creation of configuration file ' + self.ConfigTempFile)
def setUp(self):
def callback(ctx, str):
pass #FIXME! if self.verbose >= 0: print "%s %s" % (ctx, str)
libxml2.registerErrorHandler(callback, "-->")
self.Config = pokerengineconfig.Config([PokerEngineConfigTestCase.TestConfDirectory, tempfile.gettempdir()])
self.ConfigTmplFile = path.join(PokerEngineConfigTestCase.TestConfDirectory, PokerEngineConfigTestCase.TestConfigTemplateFile)
self.ConfigTempFile = path.join(tempfile.gettempdir(), PokerEngineConfigTestCase.TestConfigTemporaryFile)
self.UpgradeTmplFile = path.join(PokerEngineConfigTestCase.TestUpgradeDirectory, PokerEngineConfigTestCase.TestUpgradeTemplateFile)
self.UpgradeInvalidFile = path.join(PokerEngineConfigTestCase.TestUpgradeDirectory, PokerEngineConfigTestCase.TestUpgradeInvalidFile)
if not self.CopyFile(self.ConfigTmplFile, self.ConfigTempFile):
self.fail('Error during creation of configuration file ' + self.ConfigTempFile)
def __init__(self, mode, operation, output, options):
libxml2.registerErrorHandler(xml_error_handler, None)
self.operation = operation
self.options = options
self.msg = None
self.gt = None
self.current_mode = self.load_mode(mode)()
# Prepare output
if operation == 'update':
self.out = tempfile.TemporaryFile()
elif output == '-':
self.out = sys.stdout
else:
self.out = file(output, 'w')
def __init__(self, mode, operation, output, base_path, options):
libxml2.registerErrorHandler(xml_error_handler, None)
self.operation = operation
self.options = options
self.msg = None
self.gt = None
self.current_mode = self.load_mode(mode)()
self.base_path = base_path
# Prepare output
if operation == 'update':
self.out = tempfile.TemporaryFile(encoding='utf-8')
elif output == '-':
self.out = sys.stdout
else:
self.out = open(output, 'w', encoding='utf-8', buffering=1)
def register_libvirt_error_handler():
"""
Ignore libvirt error reporting, we just use exceptions
"""
def libvirt_callback(userdata, err):
ignore = userdata
ignore = err
def libxml2_callback(userdata, err):
ignore = userdata
logging.debug("libxml2 callback error: %s", err)
libvirt.registerErrorHandler(f=libvirt_callback, ctx=None)
import libxml2
libxml2.registerErrorHandler(f=libxml2_callback, ctx=None)
def run_test(desc,
docpath,
catalog,
exp_status="verified",
exp_err=[],
test_callback=None,
root_name="root",
root_content="replacement text"):
opts = libxml2.XML_PARSE_DTDLOAD | libxml2.XML_PARSE_NONET | libxml2.XML_PARSE_COMPACT
actual_err = []
def my_global_error_cb(ctx, msg):
actual_err.append((-1, msg))
def my_ctx_error_cb(arg, msg, severity, reserved):
actual_err.append((severity, msg))
libxml2.registerErrorHandler(my_global_error_cb, None)
try:
parser = libxml2.createURLParserCtxt(docpath, opts)
parser.setErrorHandler(my_ctx_error_cb, None)
if catalog is not None:
parser.addLocalCatalog(catalog)
if test_callback is not None:
test_callback()
parser.parseDocument()
doc = parser.doc()
actual_status = "loaded"
e = doc.getRootElement()
if e.name == root_name and e.content == root_content:
actual_status = "verified"
doc.freeDoc()
except libxml2.parserError:
actual_status = "not loaded"
if actual_status != exp_status:
print("Test '%s' failed: expect status '%s', actual '%s'" %
(desc, exp_status, actual_status))
sys.exit(1)
elif actual_err != exp_err:
print("Test '%s' failed" % desc)
print("Expect errors:")
for s, m in exp_err:
print(" [%2d] '%s'" % (s, m))
print("Actual errors:")
for s, m in actual_err:
print(" [%2d] '%s'" % (s, m))
sys.exit(1)
def __init__(self, xmlFile, addComments = False): '''Class constructor. Collects some information needed for config generation, loads XML. Keyword arguments: addComments -- add comments to result XML (default False) xmlFile -- file that contains XML to parse ''' # get path to script self.scriptPath = os.path.dirname(sys.argv[0]) # default options self.addComments = addComments self.xmlFile = xmlFile # suppress all error messages from libxml2 libxml2.registerErrorHandler(self.noErr, None) # load xml self.xml = libxml2.readFile(xmlFile, None, 0)
def __init__(self, xmlFile, addComments=False):
'''Class constructor. Collects some information needed for config generation, loads XML.
Keyword arguments:
addComments -- add comments to result XML (default False)
xmlFile -- file that contains XML to parse
'''
# get path to script
self.scriptPath = os.path.dirname(os.path.realpath(sys.argv[0]))
# default options
self.addComments = addComments
self.xmlFile = xmlFile
# suppress all error messages from libxml2
libxml2.registerErrorHandler(self.noErr, None)
# load xml
self.xml = libxml2.readFile(xmlFile, None, 0)
def xmlvalidate(log):
import libxml2
from StringIO import StringIO
from random import random
prefix="...%s..." % str(random()).replace('0.','')
msg=[]
libxml2.registerErrorHandler(lambda msg,str: msg.append(str), msg)
input = libxml2.inputBuffer(StringIO(xmlEncoding.asUTF8(aString)))
reader = input.newTextReader(prefix)
reader.SetParserProp(libxml2.PARSER_VALIDATE, 1)
ret = reader.Read()
while ret == 1: ret = reader.Read()
msg=''.join(msg)
for line in msg.splitlines():
if line.startswith(prefix): log(line.split(':',4)[-1].strip())
def validate_schema(tdl_file):
# Locate relaxng schema
rng_file = None
for tryme in [
'../../docs/tdl.rng',
'../docs/tdl.rng',
'docs/tdl.rng',
'tdl.rng',
]:
if os.path.isfile(tryme):
rng_file = tryme
break
if rng_file is None:
raise Exception('RelaxNG schema file not found: tdl.rng')
# Load relaxng schema
schema = open(rng_file, 'r').read()
rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
rngs = rngp.relaxNGParse()
# Define callback for error handling
def error_cb(ctx, str):
print "%s: %s" % (ctx, str.strip())
libxml2.registerErrorHandler(error_cb, tdl_file)
# Attempt to validate
reader = libxml2.newTextReaderFilename(tdl_file)
#reader.SetParserProp(libxml2.PARSER_VALIDATE, 1)
reader.RelaxNGSetSchema(rngs)
ret = reader.Read()
while ret == 1:
ret = reader.Read()
if ret != 0:
raise Exception('Error parsing the document: %s' % tdl_file)
if reader.IsValid() != 1:
raise Exception('Document failed to validate: %s' % tdl_file)
def run_test(desc, docpath, catalog, exp_status="verified", exp_err=[], test_callback=None,
root_name="root", root_content="replacement text"):
opts = libxml2.XML_PARSE_DTDLOAD | libxml2.XML_PARSE_NONET | libxml2.XML_PARSE_COMPACT
actual_err = []
def my_global_error_cb(ctx, msg):
actual_err.append((-1, msg))
def my_ctx_error_cb(arg, msg, severity, reserved):
actual_err.append((severity, msg))
libxml2.registerErrorHandler(my_global_error_cb, None)
try:
parser = libxml2.createURLParserCtxt(docpath, opts)
parser.setErrorHandler(my_ctx_error_cb, None)
if catalog is not None:
parser.addLocalCatalog(catalog)
if test_callback is not None:
test_callback()
parser.parseDocument()
doc = parser.doc()
actual_status = "loaded"
e = doc.getRootElement()
if e.name == root_name and e.content == root_content:
actual_status = "verified"
doc.freeDoc()
except libxml2.parserError:
actual_status = "not loaded"
if actual_status != exp_status:
print("Test '%s' failed: expect status '%s', actual '%s'" % (desc, exp_status, actual_status))
sys.exit(1)
elif actual_err != exp_err:
print("Test '%s' failed" % desc)
print("Expect errors:")
for s,m in exp_err: print(" [%2d] '%s'" % (s,m))
print("Actual errors:")
for s,m in actual_err: print(" [%2d] '%s'" % (s,m))
sys.exit(1)
def main(argc):
crawl, crawl_state = crawlstate.CrawlStateFromFlags()
start_time = time.time()
libxml2.registerErrorHandler(ErrorHandler, None)
docs = dict((docid, (url, path)) for docid, url, path in crawl_state.DocsCachedLocallyIter())
print len(docs)
print time.time() - start_time
docids = docs.keys()
if FLAGS.compare:
d1, d2 = map(int, FLAGS.compare.split(','))
print CompareDocs(docs, d1, d2, noisy=True)
return
elif FLAGS.findlike:
d1 = FLAGS.findlike
FindMoreLike(crawl_state, d1, docs, noisy=True)
elif FLAGS.cluster:
ClusterShit(crawl_state, docs)
else:
docid_range = docids[:25]
print " %s" % " ".join("%4d" % d for d in docid_range)
for d1 in docid_range:
coverages = []
for d2 in docid_range:
coverage = CompareDocs(docs, d1, d2)
coverages.append(coverage)
coverages_row_str = " ".join("%4.2f" % x for x in coverages)
print "%4d %s %4d" % (d1, coverages_row_str, d1)
return
for docid in docids[:100]:
print docid
similarities = [(other_docid, CompareDocs(docs, docid, other_docid)) for other_docid in docids[:100]]
similarities.extend(FindMoreLike(crawl_state, docid, docs, 50, 50))
similarities.sort(reverse=True, key=itemgetter(1))
PrintSimilarities(similarities, top=10)
if expectedLineNumbersDefault != getLineNumbersDefault():
failed = 1
print "FAILED to obtain correct value for " \
"lineNumbersDefault in thread %d" % thread.get_ident()
# check ther global error handler
# (which is NOT per-thread in the python bindings)
try:
doc = libxml2.parseFile("bad.xml")
except:
pass
else:
assert "failed"
# global error handler
eh = ErrorHandler()
libxml2.registerErrorHandler(eh.handler,"")
# set on the main thread only
libxml2.lineNumbersDefault(1)
test(1)
ec = len(eh.errors)
if ec == 0:
print "FAILED: should have obtained errors"
sys.exit(1)
ts = []
for i in range(THREADS_COUNT):
# expect 0 for lineNumbersDefault because
# the new value has been set on the main thread only
ts.append(Thread(target=test,args=(0,)))
for t in ts:
<item>1000</item>
</foo>"""
err=""
expect="""RNG validity error: file error line 3 element text
Type byte doesn't allow value '1000'
RNG validity error: file error line 3 element text
Error validating datatype byte
RNG validity error: file error line 3 element text
Element item failed to validate content
"""
def callback(ctx, str):
global err
err = err + "%s" % (str)
libxml2.registerErrorHandler(callback, "")
f = StringIO.StringIO(docstr)
input = libxml2.inputBuffer(f)
reader = input.newTextReader("error")
reader.RelaxNGSetSchema(rngs)
ret = reader.Read()
while ret == 1:
ret = reader.Read()
if ret != 0:
print "Error parsing the document"
sys.exit(1)
if reader.IsValid() != 0:
print "Document failed to detect the validation error"
def setUp(self):
libxml2.registerErrorHandler(xml_error_handler,None)
ctxt = libxml2.createFileParserCtxt("valid.xml")
ctxt.validate(1)
ctxt.parseDocument()
doc = ctxt.doc()
valid = ctxt.isValid()
doc.freeDoc()
if valid != 1:
print "validity check failed"
sys.exit(1)
i = i - 1
#desactivate error messages from the validation
def noerr(ctx, str):
pass
libxml2.registerErrorHandler(noerr, None)
ctxt = libxml2.createFileParserCtxt("invalid.xml")
ctxt.validate(1)
ctxt.parseDocument()
doc = ctxt.doc()
valid = ctxt.isValid()
if doc.name != "invalid.xml":
print "doc.name failed"
sys.exit(1)
root = doc.children
if root.name != "doc":
print "root.name failed"
sys.exit(1)
if valid != 0:
print "validity chec failed"
Copyright (C) 2012 "University of Twente". All Rights Reserved.
Authors: Almer Tigelaar
Djoerd Hiemstra
"""
import libxml2
import json
import re
import httplib
# local import
import snipdata
# libxml2 options
libxml2.registerErrorHandler(lambda *args: None, None)
HTML_PARSE_OPTIONS = libxml2.HTML_PARSE_RECOVER + libxml2.HTML_PARSE_NOERROR + libxml2.HTML_PARSE_NOWARNING
XML_PARSE_OPTIONS = libxml2.XML_PARSE_RECOVER + libxml2.XML_PARSE_NSCLEAN
# supported formats
FORMAT_RSS = ("//item", "title", "link", "description", ".//media:thumbnail", "Date{pubDate}", None)
FORMAT_ATOM = ("//entry", "title", "link", "summary", ".//media:thumbnail", "Date{updated}", None)
FORMAT_XMLSUGGEST = ("//Item", "Text", "Url", "Description", "Image", None, None)
FORMAT_HTML = (None, "(.//a)[1]", "(.//a)[1]/@href", None, None, None, None)
FORMAT_JSONSUGGEST = ("$.*[1]", "@", None, None, None, None, None) #jsonpath
FORMAT_NONE = (None, None, None, None, None, None, None)
# default headers
SNIPDEX_DEFAULT_HEADERS = ['Connection: close\r\n', 'User-Agent: SnipDex/0.2 (+http://www.snipdex.net/)\r\n','Accept-Encoding: identity\r\n', 'Accept-Charset: UTF-8;q=0.7,*;q=0.7\r\n', 'Cache-Control: no-cache\r\n', 'Accept-Language: nl,en;q=0.7,en-us;q=0.3\r\n', 'Referer: http://www.snipdex.net/\r\n']
remote_interesting_image_fname_prefix=sys.argv[4] print_debug=False thread_local_data=threading.local() threads_stop_flag=False touristy_spots=[] touristy_spots_lock=threading.Lock() slots_per_degree=(2**mintreelevel) ######################################################################## ######################## Panoramio API access ########################## ######################################################################## libxml2.registerErrorHandler(lambda x,y:x,None) panoramio_hostname='www.panoramio.com' panoramio_ipaddr=socket.gethostbyname(panoramio_hostname) def do_http_request(url): global thread_local_data,threads_stop_flag global panoramio_hostname,panoramio_ipaddr for try_nr in range(10): if threads_stop_flag: return None try: if not hasattr(thread_local_data,'conn'): thread_local_data.conn=httplib.HTTPConnection( panoramio_ipaddr)
#
# On the Apache configuration, make sure you have php support enabled
#
import MySQLdb
import libxml2
import sys
import string
import os
#
# We are not interested in parsing errors here
#
def callback(ctx, str):
return
libxml2.registerErrorHandler(callback, None)
#
# The dictionnary of tables required and the SQL command needed
# to create them
#
TABLES={
"symbols" : """CREATE TABLE symbols (
name varchar(255) BINARY NOT NULL,
module varchar(255) BINARY NOT NULL,
type varchar(25) NOT NULL,
descr varchar(255),
UNIQUE KEY name (name),
KEY module (module))""",
"words" : """CREATE TABLE words (
name varchar(50) BINARY NOT NULL,
def load_mode(modename):
#import imp
#found = imp.find_module(modename, submodes_path)
#module = imp.load_module(modename, found[0], found[1], found[2])
try:
sys.path.append(submodes_path)
module = __import__(modename)
modeModule = '%sXmlMode' % modename
return getattr(module, modeModule)
except:
return None
def xml_error_handler(arg, ctxt):
pass
libxml2.registerErrorHandler(xml_error_handler, None)
# Main program start
if __name__ != '__main__': raise NotImplementedError
# Parameters
submodes_path = os.path.dirname(os.path.realpath(sys.argv[0])) + "/xml2po-modes"
default_mode = 'docbook'
filename = ''
origxml = ''
mofile = ''
gt = None
ultimate = [ ]
ignored = [ ]
#
# On the Apache configuration, make sure you have php support enabled
#
import MySQLdb
import libxml2
import sys
import string
import os
#
# We are not interested in parsing errors here
#
def callback(ctx, str):
return
libxml2.registerErrorHandler(callback, None)
#
# The dictionnary of tables required and the SQL command needed
# to create them
#
TABLES={
"symbols" : """CREATE TABLE symbols (
name varchar(255) BINARY NOT NULL,
module varchar(255) BINARY NOT NULL,
type varchar(25) NOT NULL,
descr varchar(255),
UNIQUE KEY name (name),
KEY module (module))""",
"words" : """CREATE TABLE words (
name varchar(50) BINARY NOT NULL,
def tearDown(self):
libxml2.registerErrorHandler(None, None)
def setUp(self):
libxml2.registerErrorHandler(xml_error_handler, None)
def libxml2_entity_loader(URL, ID, ctxt):
dprint("Cache entity loader called for %s '%s'", URL, ID)
the_cache = get_default_cache()
file = the_cache.get_resource(URL)
try:
fd = open(file)
dprint("Cache entity loader resolved to %s", file)
except:
fd = None
return fd
# don't report errors from libxml2 parsing
def libxml2_no_error_callback(ctx, str):
pass
libxml2.registerErrorHandler(libxml2_no_error_callback, "")
def initialize():
get_default_cache()
libxml2.setEntityLoader(libxml2_entity_loader)
def run_unit_tests ():
import BaseHTTPServer
import SimpleHTTPServer
import shutil
import os
import thread
import time
class test_http_handler(SimpleHTTPServer.SimpleHTTPRequestHandler):
ctxt.parseDocument()
doc = ctxt.doc()
valid = ctxt.isValid()
doc.freeDoc()
if valid != 1:
print "validity check failed"
sys.exit(1)
i = i - 1
#desactivate error messages from the validation
def noerr(ctx, str):
pass
libxml2.registerErrorHandler(noerr, None)
ctxt = libxml2.createFileParserCtxt("invalid.xml")
ctxt.validate(1)
ctxt.parseDocument()
doc = ctxt.doc()
valid = ctxt.isValid()
if doc.name != "invalid.xml":
print "doc.name failed"
sys.exit(1)
root = doc.children
if root.name != "doc":
print "root.name failed"
sys.exit(1)
if valid != 0:
print "validity chec failed"
remote_interesting_image_fname_prefix = sys.argv[4]
print_debug = False
thread_local_data = threading.local()
threads_stop_flag = False
touristy_spots = []
touristy_spots_lock = threading.Lock()
slots_per_degree = (2**mintreelevel)
########################################################################
######################## Panoramio API access ##########################
########################################################################
libxml2.registerErrorHandler(lambda x, y: x, None)
panoramio_hostname = 'www.panoramio.com'
panoramio_ipaddr = socket.gethostbyname(panoramio_hostname)
def do_http_request(url):
global thread_local_data, threads_stop_flag
global panoramio_hostname, panoramio_ipaddr
for try_nr in range(10):
if threads_stop_flag:
return None
try:
if not hasattr(thread_local_data, 'conn'):
thread_local_data.conn = httplib.HTTPConnection(
def __init__(self, path):
''' Create a new CheckList
Attributes:
path -- full pathname of the checklist file we're implementing.
Creates a new CheckList.
'''
self.filename = path # Filename of the checklist we're implementing
self.resolution = 'Needs-Reviewing' # Resolution of the checklist
self.functions = [] # List of functions available on the checklist
# Properties available on the checklist
self.properties = properties.Properties()
self.customItemsIter = None # Iter to the custom items category
self.colors = {}
self.__unspan = re.compile(r'([^<]*)(<span[^>]*>)?([^<]*)(</span>)?(.*)')
self.colorRE = re.compile('^#[A-Fa-f0-9]{6}$')
self.gconfClient = gconf.client_get_default()
self.gconfClient.add_dir(GCONFPREFIX, gconf.CLIENT_PRELOAD_NONE)
self.__init_colors('/display/pass-color')
self.__init_colors('/display/fail-color')
self.__init_colors('/display/minor-color')
self.__init_colors('/display/notes-color')
key = GCONFPREFIX + '/display/no-auto-display'
try:
self.noAutoDisplay = self.gconfClient.get_bool(key)
except gobject.GError:
self.noAutoDisplay = (
self.gconfClient.get_default_from_schema(key).get_bool())
self.gconfClient.notify_add(key, self.__change_auto_display)
libxml2.registerErrorHandler(self.__no_display_parse_error, None)
ctxt = libxml2.newParserCtxt()
try:
checkFile = ctxt.ctxtReadFile(path, None, libxml2.XML_PARSE_DTDVALID)
except libxml2.treeError:
raise error.InvalidChecklist('%s was not an XML file' % (path))
if not ctxt.isValid():
checkFile.freeDoc()
raise error.InvalidChecklist('File does not validate against '
'the checklist DTD')
root = checkFile.getRootElement()
if root.name != 'checklist':
checkFile.freeDoc()
raise error.InvalidChecklist('File is not a valid checklist '
'policy file')
if root.prop('version') != self.formatVersion:
checkFile.freeDoc()
raise error.InvalidChecklist('Checklist file is not a known '
'version')
# Extract the name and revision of the CheckList
self.name = root.prop('name')
if not self.name:
checkFile.freeDoc()
raise error.InvalidChecklist('Checklist file does not specify '
'a name for itself')
self.revision = root.prop('revision') or '0'
summary = root.xpathEval2('/checklist/summary')
if summary:
self.summary = summary[0].content
else:
checkFile.freeDoc()
raise error.InvalidChecklist('Checklist does not have a summary')
# Create GtkTreeModel struct to store info in.
gtk.TreeStore.__init__(self, gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT)
base = root.xpathEval2('/checklist/base')
if base:
# We are loading a savefile. Just load the base info.
self.baseName = base[0].prop('name')
self.baseRevision = base[0].prop('revision')
self.baseFilename = base[0].content
self.revision = int(self.revision)
else:
# We are loading an original checklist definition. Set its values
# as the base and set the CheckList info to good values.
self.baseName = self.name
self.baseRevision = self.revision
self.baseFilename = path
self.filename = None
self.revision = 0
# Extract properties from the CheckList file
props = root.xpathEval2('/checklist/properties/property')
for p in props:
propChild = p.children
propEntry = properties.PropEntry()
propEntry.valueType = p.prop('type')
while propChild:
if propChild.name == 'require':
propEntry.propType = propChild.prop('type')
requireChild = propChild.children
while requireChild:
if requireChild.name == 'arg':
propEntry.args.append(requireChild.content)
elif requireChild.name == 'function':
propEntry.function = requireChild.content
propEntry.functionType = requireChild.prop('type')
requireChild = requireChild.next
elif propChild.name == 'value':
propEntry.value = propChild.content
propChild = propChild.next
# Set the property
self.properties[p.prop('name')] = propEntry
# Extract functions for the QA menu
functions = root.xpathEval2('/checklist/functions/function')
for function in functions:
self.functions.append((function.content, function.prop('type')))
# Record each category as a toplevel in the tree
categories = root.xpathEval2('/checklist/category')
self.entries = {}
for category in categories:
newCat = self.append(None)
gtk.TreeStore.set(self, newCat,
ISITEM, False,
RESLIST, ['Needs-Reviewing', 'Pass', 'Fail'],
RESOLUTION, 'Needs-Reviewing',
OUTPUT, '',
OUTPUTLIST, {'Needs-Reviewing': '',
'Pass': '', 'Fail': ''},
SUMMARY, category.prop('name'),
TEST, None)
self.entries[category.prop('name').lower()] = newCat
# Entries are subheadings
node = category.children
while node:
if node.name == 'description':
# Set DESCRIPTION of the heading
desc = string.join(string.split(node.content))
gtk.TreeStore.set(self, newCat, DESC, desc)
elif node.name == 'entry':
entry = self.__xml_to_entry(node)
entryIter=self.append(newCat)
gtk.TreeStore.set(self, entryIter,
ISITEM, True,
DISPLAY, entry.display,
SUMMARY, entry.name,
TEST, entry.test,
DESC, entry.desc)
self.entries[entry.name.lower()] = entryIter
# Construct the resolution from multiple states
outputList={'Needs-Reviewing': ''}
resolutionList=['Needs-Reviewing']
for i in range(len(entry.states)):
name = entry.states[i]['name']
# Order is important: We pangoize as things enter
# OUTPUT, not OUTPUTLIST.
outputList[name] = entry.states[i]['output']
if name != 'Needs-Reviewing':
resolutionList.append(entry.states[i]['name'])
res = entry.state
gtk.TreeStore.set(self, entryIter,
RESLIST, resolutionList,
OUTPUTLIST, outputList,
RESOLUTION, res,
OUTPUT, self.pangoize_output(res,
outputList[res]))
else:
# DTD validation should make this ignorable.
pass
node = node.next
checkFile.freeDoc()
### FIXME: Merge code: This is pretty close to what we have in:
# __check_resolution(). We could potentially merge these two pieces
# of code together.
category = self.get_iter_root()
catIter = category
while catIter:
entryIter = self.iter_children(catIter)
newRes = 'Pass'
while entryIter:
res = self.get_value(entryIter, RESOLUTION)
if res == 'Fail':
newRes = 'Fail'
break
elif res == 'Needs-Reviewing':
newRes = 'Needs-Reviewing'
entryIter = self.iter_next(entryIter)
gtk.TreeStore.set(self, catIter, RESOLUTION, newRes)
catIter = self.iter_next(catIter)
# Instead of code we could have:
# self.__check_resolution(category, 'Pass')
newRes = 'Pass'
while category:
res = self.get_value(category, RESOLUTION)
if res == 'Fail':
newRes = 'Fail'
break
elif res == 'Needs-Reviewing':
newRes = 'Needs-Reviewing'
category = self.iter_next(category)
self.resolution = newRes
def __init__(self, path):
""" Create a new CheckList
Attributes:
path -- full pathname of the checklist file we're implementing.
Creates a new CheckList.
"""
self.filename = path # Filename of the checklist we're implementing
self.resolution = "Needs-Reviewing" # Resolution of the checklist
self.functions = [] # List of functions available on the checklist
self.properties = {} # List of properties available on the checklist
self.customItemsIter = None # Iter to the custom items category
self.colors = {}
self.__unspan = re.compile(r"([^<]*)(<span[^>]*>)?([^<]*)(</span>)?(.*)")
self.colorRE = re.compile("^#[A-Fa-f0-9]{6}$")
self.gconfClient = gconf.client_get_default()
self.gconfClient.add_dir(GCONFPREFIX, gconf.CLIENT_PRELOAD_NONE)
self.__init_colors("/display/pass-color")
self.__init_colors("/display/fail-color")
self.__init_colors("/display/minor-color")
self.__init_colors("/display/notes-color")
key = GCONFPREFIX + "/display/no-auto-display"
self.noAutoDisplay = self.gconfClient.get_bool(key)
self.gconfClient.notify_add(key, self.__change_auto_display)
libxml2.registerErrorHandler(self.__no_display_parse_error, None)
ctxt = libxml2.newParserCtxt()
try:
checkFile = ctxt.ctxtReadFile(path, None, libxml2.XML_PARSE_DTDVALID)
except libxml2.treeError:
raise error.InvalidChecklist("%s was not an XML file" % (path))
if not ctxt.isValid():
raise error.InvalidChecklist("File does not validate against " "the checklist DTD")
root = checkFile.getRootElement()
if root.name != "checklist":
raise error.InvalidChecklist("File is not a valid checklist " "policy file")
if root.prop("version") != self.formatVersion:
raise error.InvalidChecklist("Checklist file is not a known " "version")
# Extract the name and revision of the CheckList
self.name = root.prop("name")
if not self.name:
raise error.InvalidChecklist("Checklist file does not specify " "a name for itself")
self.revision = root.prop("revision") or "0"
summary = root.xpathEval2("/checklist/summary")
if summary:
self.summary = summary[0].content
else:
raise error.InvalidChecklist("Checklist does not have a summary")
# Create GtkTreeModel struct to store info in.
gtk.TreeStore.__init__(
self,
gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT,
)
base = root.xpathEval2("/checklist/base")
if base:
# We are loading a savefile. Just load the base info.
self.baseName = base[0].prop("name")
self.baseRevision = base[0].prop("revision")
self.baseFilename = base[0].content
self.revision = int(self.revision)
else:
# We are loading an original checklist definition. Set its values
# as the base and set the CheckList info to good values.
self.baseName = self.name
self.baseRevision = self.revision
self.baseFilename = path
self.filename = None
self.name += " savefile"
self.revision = 0
# Extract properties from the CheckList file
properties = root.xpathEval2("/checklist/properties/property")
for p in properties:
propChild = p.children
value = None
function = None
functionType = None
args = []
while propChild:
if propChild.name == "require":
require = propChild.prop("type")
requireChild = propChild.children
while requireChild:
if requireChild.name == "arg":
args.append(requireChild.content)
elif requireChild.name == "function":
function = requireChild.content
functionType = requireChild.prop("type")
requireChild = requireChild.next
elif propChild.name == "value":
value = propChild.content
propChild = propChild.next
# Set the property
self.properties[p.prop("name")] = self.__Property(
p.prop("type"), value, require, function, functionType, args
)
# Extract functions for the QA menu
functions = root.xpathEval2("/checklist/functions/function")
for function in functions:
self.functions.append((function.content, function.prop("type")))
# Record each category as a toplevel in the tree
categories = root.xpathEval2("/checklist/category")
self.entries = {}
for category in categories:
newCat = self.append(None)
gtk.TreeStore.set(
self,
newCat,
ISITEM,
False,
RESLIST,
["Needs-Reviewing", "Pass", "Fail"],
RESOLUTION,
"Needs-Reviewing",
OUTPUT,
"",
OUTPUTLIST,
{"Needs-Reviewing": "", "Pass": "", "Fail": ""},
SUMMARY,
category.prop("name"),
TEST,
None,
)
self.entries[category.prop("name").lower()] = newCat
# Entries are subheadings
node = category.children
while node:
if node.name == "description":
# Set DESCRIPTION of the heading
desc = string.join(string.split(node.content))
gtk.TreeStore.set(self, newCat, DESC, desc)
elif node.name == "entry":
entry = self.__xml_to_entry(node)
entryIter = self.append(newCat)
gtk.TreeStore.set(
self,
entryIter,
ISITEM,
True,
DISPLAY,
entry.display,
SUMMARY,
entry.name,
TEST,
entry.test,
DESC,
entry.desc,
)
self.entries[entry.name.lower()] = entryIter
# Construct the resolution from multiple states
outputList = {"Needs-Reviewing": ""}
resolutionList = ["Needs-Reviewing"]
for i in range(len(entry.states)):
name = entry.states[i]["name"]
# Order is important: We pangoize as things enter
# OUTPUT, not OUTPUTLIST.
outputList[name] = entry.states[i]["output"]
if name != "Needs-Reviewing":
resolutionList.append(entry.states[i]["name"])
res = entry.state
gtk.TreeStore.set(
self,
entryIter,
RESLIST,
resolutionList,
OUTPUTLIST,
outputList,
RESOLUTION,
res,
OUTPUT,
self.pangoize_output(res, outputList[res]),
)
else:
# DTD validation should make this ignorable.
pass
node = node.next
checkFile.freeDoc()
### FIXME: Merge code: This is pretty close to what we have in:
# __check_resolution(). We could potentially merge these two pieces
# of code together.
category = self.get_iter_root()
catIter = category
while catIter:
entryIter = self.iter_children(catIter)
newRes = "Pass"
while entryIter:
res = self.get_value(entryIter, RESOLUTION)
if res == "Fail":
newRes = "Fail"
break
elif res == "Needs-Reviewing":
newRes = "Needs-Reviewing"
entryIter = self.iter_next(entryIter)
gtk.TreeStore.set(self, catIter, RESOLUTION, newRes)
catIter = self.iter_next(catIter)
# Instead of code we could have:
# self.__check_resolution(category, 'Pass')
newRes = "Pass"
while category:
res = self.get_value(category, RESOLUTION)
if res == "Fail":
newRes = "Fail"
break
elif res == "Needs-Reviewing":
newRes = "Needs-Reviewing"
category = self.iter_next(category)
self.resolution = newRes
def __init__(self, path, props):
self.customItemsPath = None
self.props = props
self.addPaths = {}
libxml2.registerErrorHandler(self.__no_display_parse_error, None)
ctxt = libxml2.newParserCtxt()
checkFile = ctxt.ctxtReadFile(path, None, libxml2.XML_PARSE_DTDVALID)
if ctxt.isValid() == False:
raise Error("File does not validate against the checklist DTD")
root = checkFile.getRootElement()
if root.name != 'checklist':
raise Error("File is not a valid checklist policy file")
if root.prop('version') != _checklistFileVersion_:
raise Error("Checklist file is not a known version")
# Extract the type from the checklist tag
self.name = root.prop('name')
if not self.name:
raise Error("Checklist file does not specify a name for itself")
self.revision = root.prop('revision')
if not self.revision:
self.revision='0'
self.type = root.prop('type')
if not self.type:
self.type = 'generic'
# Store the checklist into a GtkTreeModel
self.tree = gtk.TreeStore(gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT)
# Record each category as a toplevel in the tree
categories = root.xpathEval2('/checklist/category')
self.entries = {}
for category in categories:
iter = self.tree.append(None)
self.tree.set(iter,
ISITEM, False,
MODIFIED, False,
RESLIST, ['Needs-Reviewing', 'Pass', 'Fail'],
RESOLUTION, 'Needs-Reviewing',
OUTPUT, None,
OUTPUTLIST, {'Needs-Reviewing':None,
'Pass':None, 'Fail':None},
SUMMARY, category.prop('name'))
self.entries[category.prop('name')] = iter
# Entries are subheadings
node = category.children
while node:
if node.name == 'description':
# Set DESCRIPTION of the heading
desc = string.join(string.split(node.content))
self.tree.set(iter, DESC, desc)
elif node.name == 'entry':
entry = self.__xml_to_entry(node)
entryIter=self.tree.append(iter)
self.tree.set(entryIter,
ISITEM, True,
MODIFIED, False,
DISPLAY, entry.display,
SUMMARY, entry.name,
DESC, entry.desc)
self.entries[entry.name] = entryIter
# Construct the resolution from multiple states
resolutions={'Needs-Reviewing': None}
resolutionList=['Needs-Reviewing']
for i in range(len(entry.states)):
name = entry.states[i]['name']
output = self.colorize_output(name,
entry.states[i]['output'])
resolutions[name] = output
if name != 'Needs-Reviewing':
resolutionList.append(entry.states[i]['name'])
self.tree.set(entryIter,
RESLIST, resolutionList,
OUTPUTLIST, resolutions,
RESOLUTION, 'Needs-Reviewing',
OUTPUT, resolutions['Needs-Reviewing'])
else:
# DTD validation should make this ignorable.
pass
node = node.next
checkFile.freeDoc()
# More efficient to do the stuff in the signal handlers manually
# during setup and only register them afterwards.
self.tree.connect('row-inserted', self.__added_row)
self.tree.connect('row-changed', self.__modified_row)
def __init__(self, path, props):
libxml2.registerErrorHandler(self.__noDisplayParseError, None)
ctxt = libxml2.newParserCtxt()
checkFile = ctxt.ctxtReadFile(path, None, libxml2.XML_PARSE_DTDVALID)
if ctxt.isValid() == False:
raise Error("File does not validate against the checklist DTD")
root = checkFile.getRootElement()
if root.name != "checklist":
raise Error("File is not a valid checklist policy file")
if root.prop("version") != _checklistFileVersion_:
raise Error("Checklist file is not a known version")
# Extract the type from the checklist tag
self.name = root.prop("name")
if not self.name:
raise Error("Checklist file does not specify a name for itself")
self.revision = root.prop("revision")
if not self.revision:
self.revision = "0"
self.type = root.prop("type")
if not self.type:
self.type = "generic"
# Store the checklist into a GtkTreeModel
self.tree = gtk.TreeStore(
gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT,
)
# Record each category as a toplevel in the tree
categories = root.xpathEval2("/checklist/category")
for category in categories:
iter = self.tree.append(None)
self.tree.set(
iter,
ISITEM,
False,
RESLIST,
["Needs-Reviewing", "Pass", "Fail"],
RESOLUTION,
"Needs-Reviewing",
OUTPUT,
None,
OUTPUTLIST,
{"Needs-Reviewing": None, "Pass": None, "Fail": None},
SUMMARY,
category.prop("name"),
)
# Entries are subheadings
node = category.children
while node:
if node.name == "description":
# Set DESCRIPTION of the heading
desc = string.join(string.split(node.content))
self.tree.set(iter, DESC, desc)
elif node.name == "entry":
entry = self.__xmlToEntry(node)
entryIter = self.tree.append(iter)
self.tree.set(
entryIter, ISITEM, True, DISPLAY, entry.display, SUMMARY, entry.name, DESC, entry.desc
)
# Construct the resolution from multiple states
resolutions = {"Needs-Reviewing": None}
resolutionList = ["Needs-Reviewing"]
for i in range(len(entry.states)):
name = entry.states[i]["name"]
output = entry.states[i]["output"]
if name == "Fail":
color = props.failColor
elif name == "Non-Blocker" or name == "Needs-Reviewing":
color = props.minorColor
elif name == "Pass":
color = props.passColor
else:
color = None
if color:
output = '<span foreground="' + color + '">' + output + "</span>"
resolutions[name] = output
if name != "Needs-Reviewing":
resolutionList.append(entry.states[i]["name"])
self.tree.set(
entryIter,
RESLIST,
resolutionList,
OUTPUTLIST,
resolutions,
RESOLUTION,
"Needs-Reviewing",
OUTPUT,
resolutions["Needs-Reviewing"],
)
else:
# DTD validation should make this ignorable.
pass
node = node.next
checkFile.freeDoc()
def importfile(dbh, filename, batch=False):
module_logger.info("Importing %s Batch %s", filename, batch)
status = 1
message = 'OK'
if os.path.isfile(filename) and os.path.getsize(filename) > 0:
try:
module_logger.debug("Parsing")
xml = libxml2.newTextReaderFilename(filename)
libxml2.registerErrorHandler(_xmlerror, "")
while xml.Read():
module_logger.debug('X %s %s', xml.NodeType(), xml.Name())
if xml.NodeType(
) == libxml2.XML_READER_TYPE_ELEMENT and xml.Name(
) == 'FIMSSR':
module_logger.debug("Found FIMSSR %s %s", xml.NodeType(),
xml.Name())
status, message = _process_file(dbh, xml)
if status != 1:
module_logger.debug("Status %s bailing", status)
break
except:
module_logger.exception("Parse error %s", filename)
status = -2
message = "XML Parser Error on file " + filename
finally:
module_logger.debug("Cleanup")
libxml2.cleanupParser()
else:
module_logger.error("Zero size file %s", filename)
status = -2
message = "Zero size or truncated file " + filename
if status == 1:
try:
csr = dbh.cursor()
module_logger.debug("Store filename %s",
os.path.basename(filename))
csr.execute(_sql['file_update'],
{'filename': os.path.basename(filename)})
csr.close()
module_logger.debug("Commit")
dbh.commit()
except:
module_logger.warning("Rollback")
csr.close()
dbh.rollback()
status = -1
message = "DB error"
# if not in batch the SCHD update after file
if status == 1 and not batch:
module_logger.debug("Not Batch mode, updating schd after file %s",
filename)
status, message = _schdupdate(dbh)
return status, message
def __init__(self, path):
libxml2.registerErrorHandler(self.__noDisplayParseError, None)
ctxt = libxml2.newParserCtxt()
checkFile = ctxt.ctxtReadFile(path, None, libxml2.XML_PARSE_DTDVALID)
if ctxt.isValid() == False:
raise Error("File does not validate against the checklist DTD")
root = checkFile.getRootElement()
if root.name != 'checklist':
raise Error("File is not a valid checklist policy file")
if root.prop('version') != _checklistFileVersion_:
raise Error("Checklist file is not a known version")
# Extract the type from the checklist tag
self.type = root.prop('name')
if not self.type:
raise Error("Checklist file does not specify a type in the name attribute")
self.revision = root.prop('revision')
if not self.revision:
self.revision='0'
# Store the checklist into a GtkTreeModel
self.tree = gtk.TreeStore(gobject.TYPE_BOOLEAN,
gobject.TYPE_BOOLEAN,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT,
gobject.TYPE_PYOBJECT)
# Record each category as a toplevel in the tree
categories = root.xpathEval2('/checklist/category')
for category in categories:
iter = self.tree.append(None)
self.tree.set(iter,
ISITEM, False,
RESLIST, ['Needs-Reviewing', 'Pass', 'Fail'],
RESOLUTION, 'Needs-Reviewing',
OUTPUT, None,
OUTPUTLIST, {'Needs-Reviewing':None,
'Pass':None, 'Fail':None},
SUMMARY, category.prop('name'))
# Entries are subheadings
node = category.children
while node:
if node.name == 'description':
# Set DESCRIPTION of the heading
desc = string.join(string.split(node.content))
self.tree.set(iter, DESC, desc)
elif node.name == 'entry':
entry = self.__xmlToEntry(node)
entryIter=self.tree.append(iter)
self.tree.set(entryIter,
ISITEM, True,
DISPLAY, entry.display,
SUMMARY, entry.name,
DESC, entry.desc)
# Construct the resolution from multiple states
resolutions={'Needs-Reviewing': None}
resolutionList=['Needs-Reviewing']
for i in range(len(entry.states)):
name = entry.states[i]['name']
resolutions[name] = entry.states[i]['output']
if name != 'Needs-Reviewing':
resolutionList.append(entry.states[i]['name'])
self.tree.set(entryIter,
RESLIST, resolutionList,
OUTPUTLIST, resolutions,
RESOLUTION, 'Needs-Reviewing',
OUTPUT, resolutions['Needs-Reviewing'])
else:
# DTD validation should make this ignorable.
pass
node = node.next
checkFile.freeDoc()
#found = imp.find_module(modename, submodes_path)
#module = imp.load_module(modename, found[0], found[1], found[2])
try:
sys.path.append(submodes_path)
module = __import__(modename)
modeModule = '%sXmlMode' % modename
return getattr(module, modeModule)
except:
return None
def xml_error_handler(arg, ctxt):
pass
libxml2.registerErrorHandler(xml_error_handler, None)
# Main program start
if __name__ != '__main__': raise NotImplementedError
# Parameters
submodes_path = "./"
default_mode = 'docbook'
filename = ''
origxml = ''
mofile = ''
ultimate = []
ignored = []
filenames = []
translationlanguage = ''
def tearDown(self):
libxml2.registerErrorHandler(None,None)
def start(self):
# Memory debug specific
libxml2.debugMemory(1)
# init error handler
libxml2.registerErrorHandler(noerr, None)
nb_instances_tests = 0
nb_instances_success = 0
nb_instances_failed = 0
libxml2.lineNumbersDefault(1)
#
# Error and warnng callbacks
#
def callback(ctx, str):
global log
log.write("%s%s" % (ctx, str))
libxml2.registerErrorHandler(callback, "")
#
# Resolver callback
#
resources = {}
def resolver(URL, ID, ctxt):
global resources
if resources.has_key(URL):
return (StringIO.StringIO(resources[URL]))
log.write("Resolver failure: asked %s\n" % (URL))
log.write("resources: %s\n" % (resources))
return None
"sub": "subscript",
"sup": "superscript"
}
HTML_JUMP_NODES = [
"body",
"html"
]
HTML_TO_DOCBOOK_PROPS = {
"id": "id",
"href": "url",
"width": "width",
"src": "fileref",
"data-section-id": "data-section-id"
}
libxml2.registerErrorHandler(lambda f, ctx: None, None)
class DocBookObject(object):
def __init__(self, parent = None, object_id = None, **params):
global OBJECT_IDS
self._parent = parent
self._params = params
self._saved_state = None
self._already_warned_unconverted_docbook_node_type = []
self._already_warned_unconverted_html_node_type = []
self._already_warned_unconverted_docbook_prop = []
self._already_warned_unconverted_html_prop = []