class XmlLogger(object):
def __init__(self, path):
if os.access(path, os.F_OK):
os.rename(path, path + '.bak-' + str(load_time))
self.output = open(path, 'w')
self.xml_gen = XMLGenerator(self.output, 'utf-8')
self.lock = RLock()
def start_log(self, tag, attributes):
self.doc_tag = tag
with self.lock:
self.xml_gen.startDocument()
self.xml_gen.startElement(tag, attributes)
def end_log(self):
with self.lock:
self.xml_gen.endElement(self.doc_tag)
self.xml_gen.endDocument()
@contextmanager
def element(self, name, attrs={}):
with self.lock:
attrs = dict((key, str(value)) for key, value in attrs.items())
self.text('\n')
self.xml_gen.startElement(name, attrs)
yield
self.xml_gen.endElement(name)
def text(self, text):
with self.lock:
self.xml_gen.characters(str(text))
class XmlFormat(ModelEventHandler):
def __init__(self, out):
self.xmlgen = XMLGenerator(out, 'utf-8')
def startDocument(self):
self.xmlgen.startDocument()
def startModel(self, model, attributes, **context):
logger.debug('xmlmodel.XmlFormat: model: %s, %s', model.__name__, attributes)
logger.debug('xmlmodel.XmlFormat: context: %s', context)
recordid = context.get('recordid', ('UNKNOWN', 'UNKNOWN', -1))
hwptag = context.get('hwptag', '')
logger.info('xmlmodel.XmlFormat: rec:%d %s', recordid[2], hwptag)
if model is Text:
text = attributes.pop('text')
else:
text = None
for x in startelement(context, self.xmlgen, (model, attributes)): x[0](*x[1:])
if model is Text and text is not None:
self.xmlgen.characters(text)
unparsed = context.get('unparsed', '')
if len(unparsed) > 0:
logger.debug('UNPARSED: %s', hexdump(unparsed, True))
def endModel(self, model):
self.xmlgen.endElement(model.__name__)
def endDocument(self):
self.xmlgen.endDocument()
def main():
optparser = optparse.OptionParser(usage="Usage: %prog [options] POFILE ...")
optparser.add_option("-r", "--roundtrip", help="generate round-tripable output",
action="store_true", dest="roundtrip", default=False)
optparser.add_option("-o", "--output", help="write output to XMLFILE", metavar="XMLFILE",
action="store", dest="output", default="-")
(options, args) = optparser.parse_args()
if options.output == "-":
output = sys.stdout
else:
output = open(options.output, "w")
writer = XMLGenerator(output or sys.stdout, "utf-8")
writer.startDocument()
writer.processingInstruction("xml-stylesheet", 'type="text/xsl" href="format-html.xsl"')
writer.ignorableWhitespace("\n")
writer.startElement("messages", {})
writer.ignorableWhitespace("\n")
msg = Message(writer, options.roundtrip)
for l in fileinput.input(args):
# Continuation string?
m = re.match(r'\s*"(.*)"', l)
if m:
assert(msg.current)
msg.current.append(unescape_c(m.group(1)))
continue
else:
msg.flush()
m = re.match(r'(?s)msgid "(.*)"', l)
if m:
msg.msgid = [unescape_c(m.group(1))]
msg.current = msg.msgid
m = re.match(r'(?s)msgstr "(.*)"', l)
if m:
msg.msgstr = [unescape_c(m.group(1))]
msg.current = msg.msgstr
m = re.match(r'# \s*(.*)', l)
if m:
msg.usrcomment.append(m.group(1))
m = re.match(r'#\.\s*(.*)', l)
if m:
msg.dotcomment.append(m.group(1))
m = re.match(r'#:\s*(.*)', l)
if m:
msg.reference.append(m.group(1))
m = re.match(r'#,\s*(.*)', l)
if m:
msg.flags.append(m.group(1))
msg.flush()
writer.endElement("messages")
writer.ignorableWhitespace("\n")
writer.endDocument()
def write_2(glos, filename):
from xml.sax.saxutils import XMLGenerator
from xml.sax.xmlreader import AttributesNSImpl
xdbFp = open(filename, 'wb')
fp = XMLGenerator(xdbFp, 'utf-8')
attrs = AttributesNSImpl({}, {})
fp.startElement(u'xfardic', attrs)
for t in glos.info:
fp.startElement(unicode(t[0]), attrs)
fp.characters(unicode(t[1]))
fp.endElement(unicode(t[0]))
fp.endElement(u'xfardic')
fp.startElement(u'words', attrs)
for item in glos.data:
try:
tmpXmlFile.characters(item[1])
except:
myRaise(__file__)
printAsError('While writing xdb file, an error on word "%s"'%item[0])
continue
fp.startElement(u'word', attrs)
fp.startElement(u'in', attrs)
fp.characters(unicode(item[0]))
fp.endElement(u'in')
fp.startElement(u'out', attrs)
fp.characters(unicode(item[1]))
fp.endElement(u'out')
fp.endElement(u'words')
fp.endDocument()
xdbFp.close()
def _close(self):
"""close an already opened channel"""
generator = XMLGenerator(self.worker)
generator.endElement('channel')
# XXX
self.worker.write('\n')
self.exit = True
def save(self, filename=None):
if self.filename and filename is None: filename = self.filename
if len(self.__dict__) == 0 or filename == None: return
try:
fileobj = open(filename, 'w')
except:
print 'ERROR>>> cannot write preferences into %s' % filename
return
writer = XMLGenerator(out=fileobj, encoding=self.system_encoding)
writer.startDocument()
defaults = XmlConfigParser.__dict__
items = self.__dict__.items()
items.sort()
writer.startElement('preferences', {})
writer.characters('\n')
for key, value in items:
if defaults.has_key(key) and defaults[key] == value: continue
if key in ['filename', 'app']: continue
writer.characters('\t')
writer.startElement('%s' % key, {})
str_value = path_unicode(value.__str__())
if isinstance(value, str):
str_value = "'%s'" % (escape_quote(str_value))
writer.characters(str_value)
writer.endElement('%s' % key)
writer.characters('\n')
writer.endElement('preferences')
writer.endDocument()
fileobj.close
def export(self,filename,transport):
vprint( "osm-xml export...",1)
#remember all nodes already exported
unodes = {}
fp = open(filename, "w")
x = XMLGenerator(fp, "UTF-8")
x.startDocument()
x.startElement('osm',{"version":"0.6","generator":"crazy py script"})
for w in self.ways.itervalues():
if not 'highway' in w.tags:
continue
if transport == "all" or transport == "pt":
if not (w.tags['highway']=='bus' or w.tags['highway']=='tram'):
continue
if transport == "all" or transport == "hw":
if (w.tags['highway']=='bus' or w.tags['highway']=='tram'):
continue
w.toOSM(x)
for nid in w.nds:
if nid in unodes:#already used
continue
unodes[nid]=True
if w.nds.index(nid)==0 or w.nds.index(nid)==len(w.nds)-1:
self.nodes[nid].toOSM(x,True)
else:
self.nodes[nid].toOSM(x)
x.endElement('osm')
x.endDocument()
def write_2(glos, filename):
from xml.sax.saxutils import XMLGenerator
from xml.sax.xmlreader import AttributesNSImpl
xdbFp = open(filename, 'wb')
fp = XMLGenerator(xdbFp, 'utf-8')
attrs = AttributesNSImpl({}, {})
fp.startElement('xfardic', attrs)
for key, value in glos.iterInfo():
fp.startElement(key, attrs)
fp.characters(value)
fp.endElement(key)
fp.endElement('xfardic')
fp.startElement('words', attrs)
for entry in glos:
word = entry.getWord()
defi = entry.getDefi()
try:
tmpXmlFile.characters(defi)
except:
log.exception('While writing xdb file, an error on word "%s":'%word)
continue
fp.startElement('word', attrs)
fp.startElement('in', attrs)
fp.characters(str(word))
fp.endElement('in')
fp.startElement('out', attrs)
fp.characters(str(defi))
fp.endElement('out')
fp.endElement('words')
fp.endDocument()
xdbFp.close()
class KanjiListToXmlFile:
def __init__(self, filepath):
self.file = open(filepath, "w+")
self.handler = XMLGenerator(self.file, 'utf-8')
self.handler.startDocument()
self.handler.startElement('list', {})
def __del__(self):
self.handler.endElement('list')
self.handler.endDocument()
def kanjiToXml(self, kanji):
self.handler.startElement('character', {})
for el in kanji.element:
if isinstance(kanji.element[el], unicode):
self.handler.startElement(el, {})
self.handler.characters(kanji.element[el])
self.handler.endElement(el)
elif isinstance(kanji.element[el], int):
self.handler.startElement(el, {})
self.handler.characters(str(kanji.element[el]))
self.handler.endElement(el)
elif isinstance(kanji.element[el], list):
for m in kanji.element[el]:
self.handler.startElement(el, {})
self.handler.characters(m)
self.handler.endElement(el)
self.handler.endElement('character')
self.file.write('\n')
def kanjiListToXml(self, kanjiList):
for kanji in kanjiList:
self.kanjiToXml(kanji)
class XmlWriter(AbstractXmlWriter):
def __init__(self, output):
self._output = self._create_output(output)
self._writer = XMLGenerator(self._output, encoding='UTF-8')
self._writer.startDocument()
self.closed = False
def _create_output(self, output):
return open(output, 'w') \
if isinstance(output, basestring) else output
def _start(self, name, attrs):
self._writer.startElement(name, AttributesImpl(attrs))
def _content(self, content):
self._writer.characters(content)
def _end(self, name):
self._writer.endElement(name)
def _newline(self):
self._output.write('\n')
# Workaround for http://ironpython.codeplex.com/workitem/29474
if sys.platform == 'cli':
def _escape(self, content):
return AbstractXmlWriter._escape(self, content).encode('UTF-8')
class XmlWriter(AbstractXmlWriter):
def __init__(self, path):
self.path = path
self._output = open(path, 'wb')
self._writer = XMLGenerator(self._output, 'UTF-8')
self._writer.startDocument()
self.closed = False
def start(self, name, attributes={}, newline=True):
attrs = AttributesImpl(attributes)
self._writer.startElement(name, attrs)
if newline:
self.content('\n')
def content(self, content):
if content is not None:
self._writer.characters(self._encode(content))
def end(self, name, newline=True):
self._writer.endElement(name)
if newline:
self.content('\n')
def close(self):
self._writer.endDocument()
self._output.close()
self.closed = True
def test_xmlgen_basic_empty(self):
result = StringIO()
gen = XMLGenerator(result, short_empty_elements=True)
gen.startDocument()
gen.startElement("doc", {})
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), start + "<doc/>")
def endElement(self, name, indent = True, newl = True):
self.indents.pop()
if indent:
self.ignorableWhitespace("".join(self.indents))
XMLGenerator.endElement(self, name)
if newl:
self.ignorableWhitespace(self.newl)
def __call__(self, url, data, coords, metadata):
file = StringIO.StringIO()
gen = XMLGenerator( file, 'UTF-8' )
gen.startDocument()
gen.startElement('img', {'usemap':'#map', 'src':str(url)})
gen.endElement('img')
self.writeMap( gen, data, coords, metadata )
gen.endDocument()
return file.getvalue()
def test_xmlgen_basic(self):
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), start + "<doc></doc>")
def test_xmlgen_basic():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<doc></doc>"
def output_photo_page(self):
gen = XMLGenerator(self.wfile, 'utf-8')
gen.startDocument()
gen.startElement('html', {'lang': 'en'})
html_generator = HtmlPhotoPageGenerator(gen)
html_generator.output_head()
html_generator.output_body()
gen.endElement('html')
gen.endDocument()
def write(self, filedate, id, programme):
if programme[Fields.start].hour < self.config["files"]["start_hour"]:
filedate -= timedelta(1)
if filedate < self.now:
return
if filedate not in self.files:
f = open(os.path.join(self.base, filedate.strftime("tv-%Y%m%d.xmltv")), "wb")
g = XMLGenerator(f, "UTF-8")
g.startDocument()
f.write("<!DOCTYPE tv SYSTEM \"xmltv.dtd\">\n".encode("UTF-8"))
g.startElement("tv", {"source-info-name": "Radio Times"})
f.write("\n".encode("UTF-8"))
for channel in self.config["channels"]:
self._write_element(g, "channel", ("display-name", channel["disp"] if "disp" in channel else channel["name"]), {"id": channel["id"]})
f.write("\n".encode("UTF-8"))
self.files[filedate] = (f, g)
(f, g) = self.files[filedate]
attrs = collections.OrderedDict()
attrs["channel"] = id
attrs["start"] = programme[Fields.start].strftime("%Y%m%d%H%M%S")
attrs["stop"] = programme[Fields.stop].strftime("%Y%m%d%H%M%S")
g.startElement("programme", attrs)
self._write_element(g, "title", programme[Fields.title])
self._write_element(g, "sub-title", ": ".join(filter(None, [programme[Fields.sub_title], programme[Fields.episode]])))
self._write_element(g, "desc", programme[Fields.desc])
if programme[Fields.director] or programme[Fields.cast]:
self._write_element(g, "credits", [("director", programme[Fields.director])] + [("actor", actor) for actor in programme[Fields.cast]])
self._write_element(g, "year", programme[Fields.year])
if programme[Fields.widescreen] or programme[Fields.black_and_white]:
self._write_element(g, "video", [
("aspect", "16:9" if programme[Fields.widescreen] else None),
("colour", "no" if programme[Fields.black_and_white] else None)
])
self._write_element(g, "premiere", programme[Fields.premiere])
self._write_element(g, "new", programme[Fields.new_series])
self._write_element(g, "subtitles", programme[Fields.subtitles], {"type": "teletext"})
if programme[Fields.star_rating]:
self._write_element(g, "star-rating", ("value", programme[Fields.star_rating]))
if programme[Fields.certificate]:
self._write_element(g, "rating", ("value", programme[Fields.certificate]), {"system": "BBFC"})
self._write_element(g, "category", programme[Fields.film])
self._write_element(g, "category", programme[Fields.genre])
g.endElement("programme")
f.write("\n".encode("UTF-8"))
def test_xmlgen_escaped_attr():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {"x": unicode("\\u3042", "unicode-escape")})
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + '<doc x="あ"></doc>'
def test_xmlgen_attr_escape_manydouble():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {"a": '"\'"'})
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<doc a='\"'\"'></doc>"
def test_xmlgen_attr_escape_manysingle():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {"a": "'\"'"})
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + '<doc a="\'"\'"></doc>'
def test_xmlgen_content_escape():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.characters("<huhei&")
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<doc><huhei&</doc>"
def test_xmlgen_pi():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.processingInstruction("test", "data")
gen.startElement("doc", {})
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<?test data?><doc></doc>"
def test_xmlgen_ignorable():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.ignorableWhitespace(" ")
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<doc> </doc>"
def test_xmlgen_content_empty(self):
result = self.ioclass()
gen = XMLGenerator(result, short_empty_elements=True)
gen.startDocument()
gen.startElement("doc", {})
gen.characters("huhei")
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), self.xml("<doc>huhei</doc>"))
def test_xmlgen_pi(self):
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.processingInstruction("test", "data")
gen.startElement("doc", {})
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), start + "<?test data?><doc></doc>")
def test_xmlgen_escaped_content():
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.characters(unicode("\xa0\\u3042", "unicode-escape"))
gen.endElement("doc")
gen.endDocument()
return result.getvalue() == start + "<doc>\xa0あ</doc>"
def test_xmlgen_ignorable_empty(self):
result = StringIO()
gen = XMLGenerator(result, short_empty_elements=True)
gen.startDocument()
gen.startElement("doc", {})
gen.ignorableWhitespace(" ")
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), start + "<doc> </doc>")
def test_xmlgen_content(self):
result = StringIO()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.characters("huhei")
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), start + "<doc>huhei</doc>")
def test_xmlgen_ignorable(self):
result = self.ioclass()
gen = XMLGenerator(result)
gen.startDocument()
gen.startElement("doc", {})
gen.ignorableWhitespace(" ")
gen.endElement("doc")
gen.endDocument()
self.assertEqual(result.getvalue(), self.xml("<doc> </doc>"))
# regression test for SAX 2.0
def endElement(self, name):
"""End and XML element of the given name."""
XMLGenerator.endElement(self, name)
self.write(u'\n')
class XmlItemExporter(BaseItemExporter):
"""
Exports Items in XML format to the specified file object.
"""
def __init__(self, file, **kwargs):
"""
:param file: the file-like object to use for exporting the data.
It's write method should accept bytes (a disk file opened in
binary mode, a io.BytesIO object, etc)
:param kwargs: root_element (str) – The name of root element in
the exported XML.
:param kwargs: item_element (str) – The name of each item element
in the exported XML.
A typical output of this exporter would be:
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<name>Color TV</name>
<price>1200</price>
</item>
<item>
<name>DVD player</name>
<price>200</price>
</item>
</items>
Unless overridden in the serialize_field() method, multi-valued
fields are exported by serializing each value inside a <value>
element. This is for convenience, as multi-valued fields are
very common.
For example, the item:
>>> Item(name=['John', 'Doe'], age='23')
Would be serialized as:
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<name>
<value>John</value>
<value>Doe</value>
</name>
<age>23</age>
</item>
</items>
"""
super().__init__()
self.item_element = kwargs.pop('item_element', 'item')
self.root_element = kwargs.pop('root_element', 'items')
self._configure(kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
self._xg_characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self._beautify_newline(new_item=True)
def export_item(self, item):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
self._beautify_newline(new_item=True)
def finish_exporting(self):
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self, name, serialized_value, depth):
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
if hasattr(serialized_value, 'items'):
self._beautify_newline()
for subname, value in serialized_value.items():
self._export_xml_field(subname, value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif is_listlike(serialized_value):
self._beautify_newline()
for value in serialized_value:
self._export_xml_field('value', value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, str):
self._xg_characters(serialized_value)
else:
self._xg_characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
# Workaround for https://bugs.python.org/issue17606
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
if not isinstance(serialized_value, str):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else: # pragma: no cover
def _xg_characters(self, serialized_value):
return self.xg.characters(serialized_value)
class OSMWriter(object):
def __init__(self, filename=None, fp=None):
if filename:
self.filename = filename
self.fp = open(self.filename, 'wb')
elif fp:
self.fp = fp
self.xmlfile = XMLGenerator(self.fp, 'utf-8')
self.xmlfile.startDocument()
# TODO include version
self.xmlfile.startElement("osm", {
'version': '0.6',
'generator': 'Thomas @Mapabc 20200520-0.1'
})
def close(self, close_file=True):
self.xmlfile.characters("\n")
self.xmlfile.endElement("osm")
self.xmlfile.endDocument()
if close_file:
self.fp.close()
def bounds(self, bbox):
attrs = {
'minlon': t(bbox[0]),
'minlat': t(bbox[1]),
'maxlon': t(bbox[2]),
'maxlat': t(bbox[3])
}
self.xmlfile.characters("\n")
self.xmlfile.startElement("bounds", attrs)
self.xmlfile.endElement("bounds")
def node(self, id, lat, lon, tags=None, **metadata):
tags = tags or {}
self.xmlfile.characters("\n ")
if tags == {}:
attrs = {'id': t(id), 'lat': t(lat), 'lon': t(lon)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("node", attrs)
self.xmlfile.endElement("node")
else:
attrs = {'id': t(id), 'lat': t(lat), 'lon': t(lon)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("node", attrs)
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("node")
def way(self, id, tags, nodeids, **metadata):
tags = tags or {}
nodeids = nodeids or []
self.xmlfile.characters("\n ")
attrs = {'id': t(id)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("way", attrs)
for nodeid in nodeids:
self.xmlfile.characters("\n ")
self.xmlfile.startElement("nd", {'ref': t(nodeid)})
self.xmlfile.endElement("nd")
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("way")
def relation(self, id, tags, members, **metadata):
tags = tags or {}
self.xmlfile.characters("\n ")
attrs = {'id': t(id)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("relation", attrs)
for member in members:
attrs = {}
if len(member) == 2:
type, ref = member
attrs = {'type': t(type), 'ref': t(ref)}
else:
type, ref, role = member
attrs = {'type': t(type), 'ref': t(ref), 'role': t(role)}
self.xmlfile.characters("\n ")
self.xmlfile.startElement("member", attrs)
self.xmlfile.endElement("member")
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("relation")
def store(self):
self.id = self.userid
self.version = STORAGE_FORMAT
log.debug("storing User " + self.id)
try:
outf = TransactionalFile(
User.determineFileName(self.getStoragePath()))
gen = XMLGenerator(out=outf, encoding="UTF-8")
gen.startDocument()
gen.startElement("user", {})
gen.characters("\n")
for attr in self.__slots__ + ['userid', 'name', 'password']:
if attr == "password":
gen.startElement("passwordhash", {})
gen.characters(self.password.encode("hex"))
gen.endElement("passwordhash")
gen.characters("\n")
elif attr == "links":
gen.startElement("links", {})
gen.characters("\n")
for (name, url) in self.links.items():
gen.startElement("link", {
"name": name,
"url": url
})
gen.endElement("link")
gen.characters("\n")
gen.endElement("links")
gen.characters("\n")
elif attr == "displaystrings":
gen.startElement("displaystrings", {})
gen.characters("\n")
for (name, string) in self.displaystrings.items():
gen.startElement("string", {
"name": name,
"value": (string or u"")
})
gen.endElement("string")
gen.characters("\n")
gen.endElement("displaystrings")
gen.characters("\n")
elif attr == "categories":
gen.startElement("categories", {})
gen.characters("\n")
for cat in self.categories.values():
cat.store()
cat.putInXML(gen)
gen.endElement("categories")
gen.characters("\n")
else:
# other attribute
gen.startElement(attr, {})
gen.characters(getXMLvalue(self, attr))
gen.endElement(attr)
gen.characters('\n')
gen.endElement("user")
gen.endDocument()
outf.commit()
os.chmod(outf.name, FILEPROT)
except EnvironmentError, x:
raise StorageError(str(x))
class XMLEncoderAdapter(object):
"""Adapts a WebAPIEncoder to output XML.
This takes an existing encoder and adapts it to output a simple XML format.
"""
def __init__(self, encoder, *args, **kwargs):
self.encoder = encoder
def encode(self, o, *args, **kwargs):
self.level = 0
self.doIndent = False
stream = StringIO()
self.xml = XMLGenerator(stream, settings.DEFAULT_CHARSET)
self.xml.startDocument()
self.startElement("rsp")
self.__encode(o, *args, **kwargs)
self.endElement("rsp")
self.xml.endDocument()
self.xml = None
return stream.getvalue()
def __encode(self, o, *args, **kwargs):
if isinstance(o, dict):
for key, value in six.iteritems(o):
attrs = {}
if isinstance(key, six.integer_types):
attrs['value'] = str(key)
key = 'int'
self.startElement(key, attrs)
self.__encode(value, *args, **kwargs)
self.endElement(key)
elif isinstance(o, (tuple, list)):
self.startElement("array")
for i in o:
self.startElement("item")
self.__encode(i, *args, **kwargs)
self.endElement("item")
self.endElement("array")
elif isinstance(o, six.string_types):
self.text(o)
elif isinstance(o, six.integer_types):
self.text("%d" % o)
elif isinstance(o, bool):
if o:
self.text("True")
else:
self.text("False")
elif o is None:
pass
else:
result = self.encoder.encode(o, *args, **kwargs)
if result is None:
raise TypeError("%r is not XML serializable" % (o,))
return self.__encode(result, *args, **kwargs)
def startElement(self, name, attrs={}):
self.addIndent()
self.xml.startElement(name, attrs)
self.level += 1
self.doIndent = True
def endElement(self, name):
self.level -= 1
self.addIndent()
self.xml.endElement(name)
self.doIndent = True
def text(self, value):
self.xml.characters(value)
self.doIndent = False
def addIndent(self):
if self.doIndent:
self.xml.ignorableWhitespace('\n' + ' ' * self.level)
xg.startElement("note", {})
xg.characters(glyph.note)
xg.endElement("note")
xg.endElement("glyph")
# KLUDGE for Awami:
newName = fl.font.file_name.replace(" ", "")
newName2 = newName.replace(".VFB", ".vfb")
outname = newName2.replace(".vfb", "_AP.xml")
if outname[25:31] == "AWAMIN" or outname[25:28] == "AWB":
outname = "AwamiNastaliqRegular_AP.xml"
###############
fh = open(outname, "w")
print "filename =", outname
xg = XMLGenerator(fh, "utf-8")
xg.startDocument()
xg.startElement("font", {
'name': unicode(fl.font.font_name),
"upem": unicode(fl.font.upm)
})
for i in range(0, len(fl.font.glyphs)):
print_glyph(fl.font, fl.font.glyphs[i], i)
xg.endElement("font")
xg.endDocument()
fh.close()
print 'done'
class SeqXmlWriter(SequenceWriter):
"""Writes SeqRecords into seqXML file.
SeqXML requires the sequence alphabet be explicitly RNA, DNA or protein,
i.e. an instance or subclass of Bio.Alphabet.RNAAlphabet,
Bio.Alphabet.DNAAlphabet or Bio.Alphabet.ProteinAlphabet.
"""
def __init__(self,
target,
source=None,
source_version=None,
species=None,
ncbiTaxId=None):
"""Create Object and start the xml generator.
Arguments:
- target - Output stream opened in binary mode, or a path to a file.
- source - The source program/database of the file, for example
UniProt.
- source_version - The version or release number of the source
program or database from which the data originated.
- species - The scientific name of the species of origin of all
entries in the file.
- ncbiTaxId - The NCBI taxonomy identifier of the species of origin.
"""
super().__init__(target, "wb")
handle = self.handle
self.xml_generator = XMLGenerator(handle, "utf-8")
self.xml_generator.startDocument()
self.source = source
self.source_version = source_version
self.species = species
self.ncbiTaxId = ncbiTaxId
def write_header(self):
"""Write root node with document metadata."""
attrs = {
"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
"xsi:noNamespaceSchemaLocation":
"http://www.seqxml.org/0.4/seqxml.xsd",
"seqXMLversion": "0.4",
}
if self.source is not None:
attrs["source"] = self.source
if self.source_version is not None:
attrs["sourceVersion"] = self.source_version
if self.species is not None:
if not isinstance(self.species, str):
raise TypeError("species should be of type string")
attrs["speciesName"] = self.species
if self.ncbiTaxId is not None:
if not isinstance(self.ncbiTaxId, (str, int)):
raise TypeError("ncbiTaxID should be of type string or int")
attrs["ncbiTaxID"] = self.ncbiTaxId
self.xml_generator.startElement("seqXML", AttributesImpl(attrs))
def write_record(self, record):
"""Write one record."""
if not record.id or record.id == "<unknown id>":
raise ValueError("SeqXML requires identifier")
if not isinstance(record.id, str):
raise TypeError("Identifier should be of type string")
attrb = {"id": record.id}
if ("source" in record.annotations
and self.source != record.annotations["source"]):
if not isinstance(record.annotations["source"], str):
raise TypeError("source should be of type string")
attrb["source"] = record.annotations["source"]
self.xml_generator.startElement("entry", AttributesImpl(attrb))
self._write_species(record)
self._write_description(record)
self._write_seq(record)
self._write_dbxrefs(record)
self._write_properties(record)
self.xml_generator.endElement("entry")
def write_footer(self):
"""Close the root node and finish the XML document."""
self.xml_generator.endElement("seqXML")
self.xml_generator.endDocument()
def _write_species(self, record):
"""Write the species if given (PRIVATE)."""
local_ncbi_taxid = None
if "ncbi_taxid" in record.annotations:
local_ncbi_taxid = record.annotations["ncbi_taxid"]
if isinstance(local_ncbi_taxid, list):
# SwissProt parser uses a list (which could cope with chimeras)
if len(local_ncbi_taxid) == 1:
local_ncbi_taxid = local_ncbi_taxid[0]
elif len(local_ncbi_taxid) == 0:
local_ncbi_taxid = None
else:
ValueError(
"Multiple entries for record.annotations['ncbi_taxid'], %r"
% local_ncbi_taxid)
if "organism" in record.annotations and local_ncbi_taxid:
local_org = record.annotations["organism"]
if not isinstance(local_org, str):
raise TypeError("organism should be of type string")
if not isinstance(local_ncbi_taxid, (str, int)):
raise TypeError("ncbiTaxID should be of type string or int")
# The local species definition is only written if it differs from the global species definition
if local_org != self.species or local_ncbi_taxid != self.ncbiTaxId:
attr = {"name": local_org, "ncbiTaxID": str(local_ncbi_taxid)}
self.xml_generator.startElement("species",
AttributesImpl(attr))
self.xml_generator.endElement("species")
def _write_description(self, record):
"""Write the description if given (PRIVATE)."""
if record.description:
if not isinstance(record.description, str):
raise TypeError("Description should be of type string")
description = record.description
if description == "<unknown description>":
description = ""
if len(record.description) > 0:
self.xml_generator.startElement("description",
AttributesImpl({}))
self.xml_generator.characters(description)
self.xml_generator.endElement("description")
def _write_seq(self, record):
"""Write the sequence (PRIVATE).
Note that SeqXML requires a DNA, RNA or protein alphabet.
"""
if isinstance(record.seq, UnknownSeq):
raise TypeError(
"Sequence type is UnknownSeq but SeqXML requires sequence")
seq = str(record.seq)
if not len(seq) > 0:
raise ValueError("The sequence length should be greater than 0")
# Get the base alphabet (underneath any Gapped or StopCodon encoding)
alpha = Alphabet._get_base_alphabet(record.seq.alphabet)
if isinstance(alpha, Alphabet.RNAAlphabet):
seqElem = "RNAseq"
elif isinstance(alpha, Alphabet.DNAAlphabet):
seqElem = "DNAseq"
elif isinstance(alpha, Alphabet.ProteinAlphabet):
seqElem = "AAseq"
else:
raise ValueError("Need a DNA, RNA or Protein alphabet")
self.xml_generator.startElement(seqElem, AttributesImpl({}))
self.xml_generator.characters(seq)
self.xml_generator.endElement(seqElem)
def _write_dbxrefs(self, record):
"""Write all database cross references (PRIVATE)."""
if record.dbxrefs is not None:
for dbxref in record.dbxrefs:
if not isinstance(dbxref, str):
raise TypeError("dbxrefs should be of type list of string")
if dbxref.find(":") < 1:
raise ValueError(
"dbxrefs should be in the form ['source:id', 'source:id' ]"
)
dbsource, dbid = dbxref.split(":", 1)
attr = {"source": dbsource, "id": dbid}
self.xml_generator.startElement("DBRef", AttributesImpl(attr))
self.xml_generator.endElement("DBRef")
def _write_properties(self, record):
"""Write all annotations that are key value pairs with values of a primitive type or list of primitive types (PRIVATE)."""
for key, value in record.annotations.items():
if key not in ("organism", "ncbi_taxid", "source"):
if value is None:
attr = {"name": key}
self.xml_generator.startElement("property",
AttributesImpl(attr))
self.xml_generator.endElement("property")
elif isinstance(value, list):
for v in value:
if isinstance(value, (int, float, str)):
attr = {"name": key, "value": v}
self.xml_generator.startElement(
"property", AttributesImpl(attr))
self.xml_generator.endElement("property")
elif isinstance(value, (int, float, str)):
attr = {"name": key, "value": str(value)}
self.xml_generator.startElement("property",
AttributesImpl(attr))
self.xml_generator.endElement("property")
class SequenceWriter(object):
def __init__(self, fname, sequence, strict, report_recv, report_sent, \
loopcnt, inter):
self.fname = fname
self.seq = sequence
self.attr_loopcnt = loopcnt
self.attr_inter = inter
self.attr_strict = strict
self.attr_recv = report_recv
self.attr_sent = report_sent
def save(self):
for i in self.save_async():
pass
def save_async(self):
output = open(self.fname, 'w')
self.depth_idx = 0
self.writer = XMLGenerator(output, 'utf-8')
self.writer.startDocument()
attr_vals = {
'report_recv': (self.attr_recv) and '1' or '0',
'report_sent': (self.attr_sent) and '1' or '0',
'strict': (self.attr_strict) and '1' or '0'
}
if isinstance(self.attr_loopcnt, int):
attr_vals['loopcnt'] = str(self.attr_loopcnt)
if isinstance(self.attr_inter, (float, int)):
attr_vals['inter'] = str(self.attr_inter)
attrs = AttributesImpl(attr_vals)
self.startElement('PMScapySequence', attrs)
self.current_node = None
idx = 0
slen = float(len(self.seq))
for node in self.seq.get_children():
self.current_node = node
self.write_node(node)
idx += 1
yield idx, (idx / slen) * 100.0, output.tell()
self.current_node = None
self.endElement('PMScapySequence')
self.writer.endDocument()
output.close()
def writeSpaces(self, prepend='', append=''):
idx = max(self.depth_idx - 1, 0)
txt = '%s%s%s' % (prepend, ' ' * idx, append)
if txt:
self.writer.characters(txt)
def startElement(self, name, attrs, indent=True):
if indent:
self.writer.characters('\n')
self.depth_idx += 1
self.writeSpaces()
self.writer.startElement(name, attrs)
def endElement(self, name, indent=True):
if indent:
self.writeSpaces('\n')
self.writer.endElement(name)
self.depth_idx -= 1
def write_node(self, node):
self.start_xml_node(node.get_data())
for child_node in node.get_children():
self.current_node = child_node
self.write_node(child_node)
self.end_xml_node()
def start_xml_node(self, seq_packet):
inter = seq_packet.inter
filter = seq_packet.filter
attr_vals = {'interval': str(inter), 'filter': filter or ''}
attrs = AttributesImpl(attr_vals)
self.startElement('SequencePacket', attrs)
self.start_xml_packet(seq_packet.packet)
def end_xml_node(self):
self.endElement('SequencePacket')
def start_xml_packet(self, metapacket):
protocols = metapacket.get_protocols()
for proto in protocols:
attr_vals = {
'id': get_proto_name(proto),
'time': "%.6f" % proto.time
}
attrs = AttributesImpl(attr_vals)
self.startElement('proto', attrs)
for field in get_proto_fields(proto):
if is_default_field(proto, field):
continue
name = get_field_name(field)
value = get_field_value(proto, field)
attr_vals = {'id': name}
self.writer.characters('\n')
attrs = AttributesImpl(attr_vals)
self.startElement('field', attrs, False)
self.writer.characters(b64encode(str(value)))
self.endElement('field', False)
for idx in xrange(len(protocols)):
self.endElement('proto')
def convert_from_qs_to_xml_file(qsfile, xmlfile=None):
if(check_if_string(qsfile) and re.findall("^(http|https)\:\/\/", qsfile)):
qsheaders = {'User-Agent': useragent_string};
tree = urlparse.parse_qs(urllib2.urlopen(urllib2.Request(qsfile, None, qsheaders)).read());
else:
if(check_if_string(qsfile)):
qsfile = open(qsfile, "r");
qsfile.seek(0);
tree = urlparse.parse_qs(qsfile.read());
qsfile.close();
bctree = tree;
bctreeln = len(bctree);
if(len(bctree['type'])<len(bctree['code']) or len(bctree['type'])==len(bctree['code'])):
bctreeln = len(bctree['type']);
if(len(bctree['code'])<len(bctree['type'])):
bctreeln = len(bctree['code']);
bctreect = 0;
bcdrawlist = [];
xmlout=StringIO();
upcxml=XMLGenerator(xmlout, "utf-8");
upcxml.startDocument();
upcxml.startElement("barcodes", {});
upcxml.characters("\n");
while(bctreect < bctreeln):
qsbarcode = {}
qsbarcode.update({"type": bctree['type'][bctreect], "code": bctree['code'][bctreect]});
try:
qsbarcode.update({"file": bctree['file'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
try:
qsbarcode.update({"size": bctree['size'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
try:
qsbarcode.update({"hideinfo": bctree['hideinfo'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
try:
qsbarcode.update({"barheight": bctree['barheight'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
try:
qsbarcode.update({"textxy": bctree['textxy'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
try:
qsbarcode.update({"color": bctree['color'][bctreect]});
except KeyError:
pass;
except IndexError:
pass;
upcxml.characters(" ");
upcxml.startElement("barcode", qsbarcode);
upcxml.endElement("barcode");
upcxml.characters("\n");
bctreect = bctreect + 1;
upcxml.endElement("barcodes");
upcxml.endDocument();
xmlout.seek(0);
if(xmlfile is not None):
xmlofile = open(xmlfile, "w+");
xmlofile.write(xmlout.read());
xmlofile.close();
return True;
if(xmlfile is None):
return xmlout.read();
class DataCiteRenderer(object):
scheme_uri_choices = {
'ORCID': 'https:/orcid.org',
'ROR': 'https://ror.org'
}
def __init__(self):
self.stream = io.StringIO()
self.xml = XMLGenerator(self.stream, 'utf-8')
def render(self, data):
self.data = data.datacite
self.render_document()
dom = parseString(self.stream.getvalue())
return dom.toprettyxml(indent=' ')
def render_node(self, tag, attrs, value):
# remove None values from attrs
attrs = dict((k, v) for k, v in attrs.items() if v)
self.xml.startElement(tag, attrs)
if value is not None:
self.xml.characters(str(value))
self.xml.endElement(tag)
def render_document(self):
self.xml.startDocument()
self.render_resource()
self.xml.endDocument()
def render_resource(self):
self.xml.startElement(
'resource', {
'xmlns:xsi':
'http://www.w3.org/2001/XMLSchema-instance',
'xmlns':
'http://datacite.org/schema/kernel-4',
'xsi:schemaLocation':
'http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4.3/metadata.xsd'
})
# identifier
identifier = next(item for item in self.data.get('identifiers', [])
if item.get('identifierType') == 'DOI')
self.render_node('identifier',
{'identifierType': identifier.get('identifierType')},
identifier.get('identifier'))
# creators
self.xml.startElement('creators', {})
for creator in self.data.get('creators', []):
self.xml.startElement('creator', {})
self.render_node('creatorName', {'nameType': 'Personal'},
creator.get('name'))
if creator.get('givenName'):
self.render_node('givenName', {}, creator.get('givenName'))
if creator.get('familyName'):
self.render_node('familyName', {}, creator.get('familyName'))
if creator.get('nameIdentifier'):
scheme = creator.get('nameIdentifierScheme', 'ORCID')
self.render_node(
'nameIdentifier', {
'nameIdentifierScheme': scheme,
'schemeURI': self.scheme_uri_choices.get(scheme)
}, creator.get('nameIdentifier'))
for affiliation in creator.get('affiliations', []):
if affiliation.get('affiliationIdentifier'):
affiliation_scheme = affiliation.get(
'affiliationIdentifierScheme', 'ROR')
affiliation_attrs = {
'affiliationIdentifier':
affiliation.get('affiliationIdentifier'),
'affiliationIdentifierScheme':
affiliation_scheme,
'schemeURI':
self.scheme_uri_choices.get(affiliation_scheme)
}
else:
affiliation_attrs = {}
self.render_node('affiliation', affiliation_attrs,
affiliation.get('affiliation'))
self.xml.endElement('creator')
self.xml.endElement('creators')
# titles
self.xml.startElement('titles', {})
for i, title in enumerate(self.data.get('titles', [])):
if i == 0:
self.render_node('title', {}, title.get('title'))
else:
self.render_node(
'title',
{'titleType': title.get('titleType', 'AlternativeTitle')},
title.get('title'))
self.xml.endElement('titles')
# publisher
self.render_node('publisher', {}, self.data.get('publisher'))
# publicationYear
self.render_node('publicationYear', {},
self.data.get('publicationYear'))
# subjects
if 'subjects' in self.data:
self.xml.startElement('subjects', {})
for subject in self.data.get('subjects', []):
self.render_node(
'subject', {
'subjectScheme': subject.get('subjectScheme'),
'schemeURI': subject.get('schemeURI'),
'valueURI': subject.get('valueURI')
}, subject.get('subject'))
self.xml.endElement('subjects')
# contributors
if 'contributors' in self.data:
self.xml.startElement('contributors', {})
for contributor in self.data.get('contributors', []):
self.xml.startElement(
'contributor',
{'contributorType': contributor.get('contributorType')})
self.render_node(
'contributorName',
{'nameType': contributor.get('nameType', 'Personal')},
contributor.get('name'))
if contributor.get('givenName'):
self.render_node('givenName', {},
contributor.get('givenName'))
if contributor.get('familyName'):
self.render_node('familyName', {},
contributor.get('familyName'))
if contributor.get('nameIdentifier'):
scheme = creator.get('nameIdentifierScheme', 'ORCID')
self.render_node(
'nameIdentifier', {
'nameIdentifierScheme': scheme,
'schemeURI': self.scheme_uri_choices.get(scheme)
}, contributor.get('nameIdentifier'))
for affiliation in contributor.get('affiliations', []):
scheme = affiliation.get('affiliationIdentifierScheme',
'ROR')
self.render_node(
'affiliation', {
'affiliationIdentifier':
affiliation.get('affiliationIdentifier'),
'affiliationIdentifierScheme':
scheme,
'schemeURI':
self.scheme_uri_choices.get(scheme)
}, affiliation.get('affiliation'))
self.xml.endElement('contributor')
self.xml.endElement('contributors')
# dates
if 'dates' in self.data:
self.xml.startElement('dates', {})
for date in self.data.get('dates', {}):
self.render_node('date', {'dateType': date.get('dateType')},
date.get('date'))
self.xml.endElement('dates')
# language
self.render_node('language', {}, self.data.get('language', 'eng'))
# resourceType
self.render_node(
'resourceType', {
'resourceTypeGeneral':
self.data.get('types', {}).get('resourceTypeGeneral',
'Dataset')
},
self.data.get('types', {}).get('resourceType'))
# alternateIdentifiers
if 'alternateIdentifiers' in self.data:
self.xml.startElement('alternateIdentifiers', {})
for alternate_identifier in self.data.get('alternateIdentifiers',
[]):
self.render_node(
'alternateIdentifier', {
'alternateIdentifierType':
alternate_identifier.get('alternateIdentifierType')
}, alternate_identifier.get('alternateIdentifier'))
self.xml.endElement('alternateIdentifiers')
# relatedIdentifiers
if 'relatedIdentifiers' in self.data:
self.xml.startElement('relatedIdentifiers', {})
for related_identifier in self.data.get('relatedIdentifiers', []):
if related_identifier.get('relatedIdentifier'):
self.render_node(
'relatedIdentifier', {
'relatedIdentifierType':
related_identifier.get('relatedIdentifierType'),
'relationType':
related_identifier.get('relationType')
}, related_identifier.get('relatedIdentifier'))
self.xml.endElement('relatedIdentifiers')
# version
if 'version' in self.data:
self.render_node('version', {}, self.data.get('version'))
# rightsList
if 'rightsList' in self.data:
self.xml.startElement('rightsList', {})
for rights in self.data.get('rightsList', []):
self.render_node('rights',
{'rightsURI': rights.get('rightsURI')},
rights.get('rights'))
self.xml.endElement('rightsList')
# descriptions
if 'descriptions' in self.data:
self.xml.startElement('descriptions', {})
for description in self.data.get('descriptions', []):
self.render_node(
'description', {
'descriptionType':
description.get('descriptionType', 'Abstract')
}, description.get('description'))
self.xml.endElement('descriptions')
# fundingReferences
if 'fundingReferences' in self.data:
self.xml.startElement('fundingReferences', {})
for funding_reference in self.data.get('fundingReferences', []):
self.xml.startElement('fundingReference', {})
self.render_node('funderName', {},
funding_reference.get('name'))
if 'funderIdentifier' in funding_reference:
self.render_node(
'funderIdentifier', {
'funderIdentifierType':
affiliation.get('funderIdentifierType', 'ROR'),
'schemeURI':
affiliation.get('schemeURI', 'https://ror.org')
}, funding_reference.get('funderIdentifier'))
if 'awardNumber' in funding_reference:
self.render_node(
'awardNumber',
{'awardURI': funding_reference.get('awardURI')},
funding_reference.get('awardNumber'))
if 'awardTitle' in funding_reference:
self.render_node('awardTitle', {},
funding_reference.get('awardTitle'))
self.xml.endElement('fundingReference')
self.xml.endElement('fundingReferences')
self.xml.endElement('resource')
class IndesignVisitor(NodeVisitor):
def __init__(self, document):
NodeVisitor.__init__(self, document)
self._output = StringIO()
self.generator = XMLGenerator(self._output, "utf-8")
self.generator.outf = self._output
self.listenv = None
self.tableenv = False
self.within_index = False
self.restrect_newline = True
def newline(self):
if self.restrect_newline is not True:
self.generator.outf.write("\n")
def astext(self):
return self._output.getvalue()
def visit_document(self, node):
self.generator.startDocument()
self.generator.outf.write(
'<doc xmlns:aid="http://ns.adobe.com/AdobeInDesign/4.0/">')
self.sec_level = 0
def depart_document(self, node):
self.generator.endDocument()
self.generator.outf.write("</doc>")
def visit_paragraph(self, node):
# does not print <p> in list
if not (self.listenv or self.tableenv):
self.generator.startElement('p', {})
def depart_paragraph(self, node):
if not (self.listenv or self.tableenv):
self.generator.endElement('p')
self.newline()
def visit_section(self, node):
assert not self.within_index
self.sec_level += 1
def depart_section(self, node):
self.sec_level -= 1
self.newline()
def visit_title(self, node):
level = "h" + str(self.sec_level)
self.generator.startElement('title', {"aid:pstyle": level})
def depart_title(self, node):
self.generator.endElement('title')
dtp = u'<?dtp level="{0}" section="{1}"?>'.format(
self.sec_level,
node.astext(),
)
self.generator.outf.write(dtp)
self.newline()
def visit_Text(self, node):
self.generator.characters(node.astext())
def depart_Text(self, node):
pass
def visit_index(self, node):
# self.generator.startElement('index', {})
self.within_index = True
pass
def depart_index(self, node):
# self.generator.endElement('index')
self.within_index = False
pass
def visit_target(self, node):
# self.generator.startElement('section', {})
pass
def depart_target(self, node):
pass
def visit_emphasis(self, node):
self.generator.startElement("i", {})
def depart_emphasis(self, node):
self.generator.endElement("i")
def visit_math(self, node):
pass
def depart_math(self, node):
pass
def visit_doctest_block(self, node):
self.generator.startElement("programlisting", {"language": "python"})
def depart_doctest_block(self, node):
self.generator.endElement("programlisting")
def visit_strong(self, node):
self.generator.startElement("b", {})
def depart_strong(self, node):
self.generator.endElement("b")
def visit_title_reference(self, node):
pass
def depart_title_reference(self, node):
pass
def visit_displaymath(self, node):
pass
def depart_displaymath(self, node):
pass
def visit_note(self, node):
self.generator.startElement('note', {})
def depart_note(self, node):
self.generator.endElement('note')
def visit_tip(self, node):
self.generator.startElement('tip', {})
def depart_tip(self, node):
self.generator.endElement('tip')
def visit_caution(self, node):
self.generator.startElement('caution', {})
def depart_caution(self, node):
self.generator.endElement('caution')
def visit_warning(self, node):
self.generator.startElement('warning', {})
def depart_warning(self, node):
self.generator.endElement('warning')
def visit_unknown_visit(self, node):
pass
def depart_unknown_visit(self, node):
pass
def visit_number_reference(self, node):
pass
def depart_number_reference(self, node):
pass
def visit_literal_block(self, node):
self.generator.startElement('pre', {})
if "highlight_args" in node.attributes.keys(
) and node.attributes["linenos"] == "True":
self.generator.startElement(
"listinfo", {"language": node.attributes["language"]})
self._lit_block_tag = "listinfo"
else:
self.generator.startElement("list", {'type': 'emlist'})
self._lit_block_tag = "list"
self.newline()
def depart_literal_block(self, node):
self.newline()
self.generator.endElement(self._lit_block_tag)
del self._lit_block_tag
self.generator.endElement('pre')
self.newline()
def visit_literal(self, node):
self.generator.startElement('tt', {'type': 'inline-code'})
def depart_literal(self, node):
self.generator.endElement('tt')
def visit_comment(self, node):
raise nodes.SkipNode
def depart_comment(self, node):
pass
def visit_compound(self, node):
pass
def depart_compound(self, node):
pass
def visit_compact_paragraph(self, node):
pass
def depart_compact_paragraph(self, node):
pass
def visit_figure(self, node):
pass
def depart_figure(self, node):
pass
def visit_image(self, node):
caption = None
for c in node.parent.children:
if isinstance(c, nodes.caption):
caption = c.astext()
legend = None
for c in node.parent.children:
if isinstance(c, nodes.legend):
legend = c.astext()
filename = os.path.basename(os.path.splitext(node['uri'])[0])
if node.get('inline'):
self.generator.startElement('a', {"linkurl": filename})
self.generator.endElement('a')
else:
self.generator.startElement('img', {})
self.generator.startElement('Image', {'href': filename})
self.generator.endElement('Image')
if caption:
self.generator.startElement('caption', {})
self.generator.outf.write(caption)
self.generator.endElement('caption')
if legend:
self.add_lines([legend])
self.generator.endElement('img')
raise nodes.SkipNode
def depart_image(self, node):
pass
def depart_reference(self, node):
self.generator.endElement("ref")
def visit_caption(self, node):
if node.parent.tagname == 'figure' or node.parent.tagname == 'container':
raise nodes.SkipNode
else:
pass
def depart_caption(self, node):
pass
def visit_bullet_list(self, node):
self.listenv = "ul"
self.generator.startElement("ul", {})
self.newline()
def depart_bullet_list(self, node):
self.listenv = None
self.generator.endElement("ul")
self.newline()
def visit_enumerated_list(self, node):
self.listenv = "ol"
self.generator.startElement("ol", {})
self.newline()
def depart_enumerated_list(self, node):
self.listenv = None
self.generator.endElement("ol")
def visit_reference(self, node):
atts = {'class': 'reference'}
if node.get('internal') or 'refuri' not in node:
atts['class'] += ' internal'
else:
atts['class'] += ' external'
if 'refuri' in node:
atts['href'] = node['refuri']
else:
assert 'refid' in node, \
'References must have "refuri" or "refid" attribute.'
atts['href'] = '#' + node['refid']
if not isinstance(node.parent, nodes.TextElement):
assert len(node) == 1 and isinstance(node[0], nodes.image)
atts['class'] += ' image-reference'
if 'reftitle' in node:
atts['title'] = node['reftitle']
self.generator.startElement("ref", atts)
#self.generator.endElement("ref")
# if node.get('secnumber'):
# self.body.append(('%s' + self.secnumber_suffix) %
# '.'.join(map(str, node['secnumber'])))
def visit_list_item(self, node):
style = ""
if self.listenv:
style = self.listenv + "-item"
self.generator.startElement("li", {"aid:pstyle": style})
def depart_list_item(self, node):
self.generator.endElement("li")
self.newline()
def visit_field_list(self, node):
self.generator.startElement("variablelist", {})
def depart_field_list(self, node):
self.generator.endElement("variablelist")
def visit_field(self, node):
self.generator.startElement("varlistentry", {})
def depart_field(self, node):
self.generator.endElement("varlistentry")
def visit_field_name(self, node):
self.generator.startElement("term", {})
def depart_field_name(self, node):
self.generator.endElement("term")
def visit_field_body(self, node):
self.generator.startElement("listitem", {})
def depart_field_body(self, node):
self.generator.endElement("listitem")
def visit_definition_list(self, node):
self.listenv = "dl"
self.generator.startElement("dl", {})
self.newline()
def depart_definition_list(self, node):
self.listenv = None
self.generator.endElement("dl")
self.newline()
def visit_definition_list_item(self, node):
pass
# self.generator.startElement("dt", {})
def depart_definition_list_item(self, node):
pass
# self.generator.endElement("dt")
# self.newline()
def visit_term(self, node):
pass
self.generator.startElement("dt", {})
def depart_term(self, node):
pass
self.generator.endElement("dt")
self.newline()
def visit_definition(self, node):
self.generator.startElement("dd", {})
def depart_definition(self, node):
self.generator.endElement("dd")
self.newline()
def visit_block_quote(self, node):
self.generator.startElement("quote", {})
self.newline()
def depart_block_quote(self, node):
self.generator.endElement("quote")
self.newline()
def visit_inline(self, node):
pass
# if 'xref' in node['classes'] or 'term' in node['classes']:
# self.add_text('*')
def depart_inline(self, node):
pass
# if 'xref' in node['classes'] or 'term' in node['classes']:
# self.add_text('*')
def visit_todo_node(self, node):
raise nodes.SkipNode
# self.generator.startElement("todo", {})
def depart_todo_node(self, node):
# self.generator.endElement("todo")
pass
def visit_container(self, node):
if 'literal-block-wrapper' in node['classes']:
self.generator.startElement('codelist', {})
cap_node = node.children[0]
if cap_node.tagname == 'caption':
self.generator.startElement('caption', {})
self.generator.outf.write(cap_node.astext())
self.generator.endElement('caption')
def depart_container(self, node):
if 'literal-block-wrapper' in node['classes']:
self.generator.endElement('codelist')
def visit_citation(self, node):
self.generator.startElement('a', {})
self.generator.outf.write(node.rawsource)
#print(node.__dict__)
def depart_citation(self, node):
self.generator.endElement('a')
def visit_label(self, node):
self.generator.startElement('label', {})
def depart_label(self, node):
self.generator.endElement('label')
def visit_footnote(self, node):
if 'obsolated' in node['classes']:
raise nodes.SkipNode
self.generator.startElement("footnote", {'id': node['ids'][0]})
node.children.remove(node.children[0])
#self.generator.startElement("footnote", {'id':node['ids'][0]})
def depart_footnote(self, node):
self.generator.endElement('footnote')
#self.generator.endElement("footnote")
def visit_footnote_reference(self, node):
raise nodes.SkipNode
def depart_footnote_reference(self, node):
pass
def visit_substitution_definition(self, node):
pass
def depart_substitution_definition(self, node):
pass
def visit_table(self, node):
self.tableenv = True
self.generator.startElement('table', {})
def depart_table(self, node):
self.generator.endElement('table')
self.tableenv = False
def visit_tgroup(self, node):
#self.generator.startElement('tgroup', {})
pass
def depart_tgroup(self, node):
#self.generator.endElement('tgroup')
pass
def visit_colspec(self, node):
#self.generator.startElement('colspec', {})
pass
def depart_colspec(self, node):
# self.generator.endElement('colspec')
pass
def visit_thead(self, node):
self.generator.startElement('thead', {'aid:pstyle': 'header'})
def depart_thead(self, node):
self.generator.endElement('thead')
def visit_row(self, node):
# self.generator.startElement('tr', {})
pass
def depart_row(self, node):
# self.generator.endElement('tr')
pass
def visit_entry(self, node):
self.generator.startElement('td', {
'aid:table': 'cell',
'aid:crows': '1',
'aid:ccols': '1'
})
def depart_entry(self, node):
self.generator.endElement('td')
#self.generator.outf.write('\t')
def visit_tbody(self, node):
tcol = str(node.parent.attributes['cols'])
trow = str(len(node.children))
self.generator.startElement('tbody', {
'aid:tcols': tcol,
'aid:trows': trow,
'aid:table': 'table'
})
def depart_tbody(self, node):
self.generator.endElement('tbody')
def visit_problematic(self, node):
pass
def depart_problematic(self, node):
pass
def visit_superscript(self, node):
self.generator.startElement('sup', {})
def depart_superscript(self, node):
self.generator.endElement('sup')
def visit_column(self, node):
self.generator.startElement('column', {})
self.generator.outf.write(
'<title aid:pstyle="column-title">%s</title>' % node['title'])
def depart_column(self, node):
self.generator.endElement('column')
def visit_raw(self, node):
self.generator.startElement('raw', {})
def depart_raw(self, node):
self.generator.endElement('raw')
def visit_transition(self, node):
self.generator.startElement('transition', {})
def depart_transition(self, node):
self.generator.endElement('transition')
class XmlAnnotationWriter:
def __init__(self, file):
self.version = "1.1"
self.file = file
self.xmlgen = XMLGenerator(self.file, 'utf-8')
self._level = 0
def _indent(self, newline=True):
if newline:
self.xmlgen.ignorableWhitespace("\n")
self.xmlgen.ignorableWhitespace(" " * self._level)
def _add_version(self):
self._indent()
self.xmlgen.startElement("version", {})
self.xmlgen.characters(self.version)
self.xmlgen.endElement("version")
def open_document(self):
self.xmlgen.startDocument()
def open_root(self):
self.xmlgen.startElement("annotations", {})
self._level += 1
self._add_version()
def _add_meta(self, meta):
self._level += 1
for k, v in meta.items():
if isinstance(v, OrderedDict):
self._indent()
self.xmlgen.startElement(k, {})
self._add_meta(v)
self._indent()
self.xmlgen.endElement(k)
elif isinstance(v, list):
self._indent()
self.xmlgen.startElement(k, {})
for tup in v:
self._add_meta(OrderedDict([tup]))
self._indent()
self.xmlgen.endElement(k)
else:
self._indent()
self.xmlgen.startElement(k, {})
self.xmlgen.characters(v)
self.xmlgen.endElement(k)
self._level -= 1
def add_meta(self, meta):
self._indent()
self.xmlgen.startElement("meta", {})
self._add_meta(meta)
self._indent()
self.xmlgen.endElement("meta")
def open_track(self, track):
self._indent()
self.xmlgen.startElement("track", track)
self._level += 1
def open_image(self, image):
self._indent()
self.xmlgen.startElement("image", image)
self._level += 1
def open_box(self, box):
self._indent()
self.xmlgen.startElement("box", box)
self._level += 1
def open_polygon(self, polygon):
self._indent()
self.xmlgen.startElement("polygon", polygon)
self._level += 1
def open_polyline(self, polyline):
self._indent()
self.xmlgen.startElement("polyline", polyline)
self._level += 1
def open_points(self, points):
self._indent()
self.xmlgen.startElement("points", points)
self._level += 1
def open_cuboid(self, cuboid):
self._indent()
self.xmlgen.startElement("cuboid", cuboid)
self._level += 1
def open_tag(self, tag):
self._indent()
self.xmlgen.startElement("tag", tag)
self._level += 1
def add_attribute(self, attribute):
self._indent()
self.xmlgen.startElement("attribute", {"name": attribute["name"]})
self.xmlgen.characters(attribute["value"])
self.xmlgen.endElement("attribute")
def close_box(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("box")
def close_polygon(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("polygon")
def close_polyline(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("polyline")
def close_points(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("points")
def close_cuboid(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("cuboid")
def close_tag(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("tag")
def close_image(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("image")
def close_track(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("track")
def close_root(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("annotations")
self._indent()
def close_document(self):
self.xmlgen.endDocument()
def endElement(self, name):
self.depth -= 1
if not self.chars:
self.characters('\n' + ' ' * self.depth)
XMLGenerator.endElement(self, name)
self.chars = False
class SpchToXml:
def __init__(self, outfh=sys.stdout):
self.h5file = None
# build xml structure
self.outfh = outfh
self.xmldoc = XMLGenerator(outfh, 'utf-8')
self.xmldoc.startElement(u'SOLiD', AttributesImpl({}))
self.haverundata = False
self.xmlpanels = None
self.filterTag = None
def setFilterTag(self,tag):
if tag != None:
self.filterTag = tag.upper()
def openHDF(self, fname):
if self.h5file != None:
self.closeHDF()
self.h5file = tables.openFile(fname, mode = "r", PYTABLES_SYS_ATTRS=False)
def closeHDF(self):
if self.h5file != None:
self.h5file.close()
self.h5file = None
def processSPCH(self, spch):
self.openHDF(spch)
if not self.haverundata:
self.processHeader()
self.processPanel()
self.closeHDF()
def processHeader(self):
self.walkGroup(self.getGroup("/Header"))
self.processLeaf(self.getGroup("/Description"))
self.haverundata = True
log("Processed Header and Description",2)
# After header, start panel block
self.xmldoc.startElement(u'Panels', AttributesImpl({}))
def processPanel(self):
panelsgroup = self.getGroup("/Panels")
# Don't want to walk panels, but do want to walk all panels.
for node in self.h5file.iterNodes(panelsgroup, 'Group'):
self.walkGroup(node, 'panel_' + node._v_name)
def getGroup(self, name="/"):
return self.h5file.getNode(name)
def walkGroup(self, group, name=None):
log("Group: " + str(group),2)
if name == None:
name = group._v_name
# SPCH specific mutation of node name to valid xml tag name
# 01_V1 - xml node names must not begin with numeric value
cyclere = re.compile("^\d{2}_V\d+$")
if cyclere.match(name):
name = u"C_" + name
# start group add add attributes
# NB calls startElement on name
self.buildNodeWithAttrs(group, name, group._v_attrs)
# Walk leaf nodes before child groups
for leaf in self.h5file.iterNodes(group, 'Leaf'):
self.processLeaf(leaf)
# Recursively walk child groups
for subgroup in self.h5file.iterNodes(group, 'Group'):
if(self.filterTag and subgroup._v_name in self.filterTag):
continue
self.walkGroup(subgroup)
self.xmldoc.endElement(unicode(name))
self.xmldoc._out.write("\n")
def buildNodeWithAttrs(self, node, name, attrs):
""" Returns attribute node, plus children """
(nodeattr, childnodes) = self.processAttrs(node, attrs)
xmlattr = AttributesImpl(nodeattr)
self.xmldoc.startElement(name, xmlattr)
for (name, arr) in childnodes.iteritems():
(s, value) = arr
self.xmldoc.startElement(unicode(name), AttributesImpl({u'shape': unicode(s)}))
self.xmldoc.characters(unicode(value))
self.xmldoc.endElement(unicode(name))
self.xmldoc._out.write("\n")
def processAttrs(self, node, attrs):
if attrs == None:
return ({}, {})
xmlattr = {}
childnodes = {}
for attrname in attrs._g_listAttr(node):
try:
attrvalue = attrs._g_getAttr(node, attrname)
log("Attr '%s': %s" % (attrname, attrvalue) ,1)
if attrvalue is None:
continue
if attrvalue.shape == () or (len(attrvalue.shape) == 1 and attrvalue.shape[0] == 1):
xmlattr[unicode(attrname)] = unicode(str(attrvalue[0]))
else:
childnodes[attrname] = self.processArray(attrvalue)
except TypeError:
raise
# Unable to handle unimplemented attribute types - var string
pass
return (xmlattr, childnodes)
def processArray(self, arr):
s = arr.shape
data = None
if len(s) > 1:
# NB - The column major data saved by matlab in hdf5 is read as row major by numpy/pytables
# This transpose corrects for that discrepancy and converts to true row major
ret = numpy.transpose(arr)
if ret != None:
# NB 2.3/numarray will return, 2.5/numpy will transpose in place and return none
arr = ret
data = arr.flat
else:
data = arr
strdata = " ".join(map(repr,data))
return (s, strdata)
def processLeaf(self, leaf):
log("Leaf: %s" % str(leaf),1)
attrs = leaf.attrs
if (attrs == None) and (leaf.shape == ()):
return
self.buildNodeWithAttrs(leaf, leaf.name, leaf.attrs)
if (isinstance(leaf, tables.Table) or isinstance(leaf, tables.Array)) and leaf.shape != ():
# Output any data fields with max size 4x4
log("Data: %s" % ( repr(leaf) ), 1)
#log("Data: %s" % ( repr(leaf[0])), 1)
if len(leaf.shape) == 1 and leaf.shape[0] == 1:
self.xmldoc.characters(unicode(str(leaf[0])))
elif len(leaf.shape) <= 2 and max(leaf.shape) <= 4:
log("Leaf with array: %s" % str(leaf),2)
#(s, data) = self.processArray(leaf)
#self.xmldoc.startElement(u'data', {u'shape': unicode(s) })
#self.xmldoc.characters(unicode(data))
#self.xmldoc.endElement(u'data')
# NB This is usually a mistake - only happens when numbeads = 0
elif (isinstance(leaf, tables.UnImplemented)):
log("Unimplemented: %s" % ( repr(leaf) ), 1)
# Ignore unimplemented node
else:
log("ERROR: Unsupported HDF5 Node type: " + str(leaf), 4)
self.xmldoc.endElement(unicode(leaf.name))
self.xmldoc._out.write("\n")
def close(self):
self.xmldoc._out.write("\n")
self.xmldoc.endElement(u'Panels')
self.xmldoc.endElement(u'SOLiD')
self.xmldoc.endDocument()
self.xmldoc._out.write("\n")
#self.outfh.close()
self.outfh = None
def end_test(self, result):
"""Called after each test."""
failures = errors = skipped = disabled = 0
if result.eofs == 'O':
pass
elif result.eofs == 'S':
skipped = 1
elif result.eofs == 'F':
failures = 1
elif result.eofs == 'E':
errors = 1
else:
errors = 1
classname = encode(result.case.uid.replace("/", "."))
if classname.endswith('.py'):
classname = classname[:-3]
filename = os.path.join(self.outdir, '%s.xml' % (result.case.uid, ))
dirname = os.path.dirname(filename)
try:
os.makedirs(dirname)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
f_report = open(filename, 'w')
try:
xml = XMLGenerator(f_report, encoding='utf-8')
xml.startDocument()
xml.startElement(
'testsuite', {
'name': classname,
'tests': '%d' % (1, ),
'failures': '%d' % (failures, ),
'errors': '%d' % (errors, ),
'time': '%0.3f' % (result.duration / 1000.0, ),
'disabled': '%d' % (disabled, ),
'skipped': '%d' % (skipped, ),
'timestamp': self.now.isoformat(),
'hostname': os.uname()[1],
})
xml.startElement('properties', {})
xml.startElement('property', {
'name': 'hostname',
'value': result.environment.hostname,
})
xml.endElement('property')
xml.startElement('property', {
'name': 'architecture',
'value': result.environment.architecture,
})
xml.endElement('property')
xml.startElement('property', {
'name': 'role',
'value': result.environment.role,
})
xml.endElement('property')
xml.startElement(
'property', {
'name': 'version',
'value': '%s' % (result.environment.ucs_version, ),
})
xml.endElement('property')
if result.case.description:
xml.startElement(
'property', {
'name':
'description',
'value':
encode(result.case.description or result.case.uid),
})
xml.endElement('property')
xml.endElement('properties')
xml.startElement(
'testcase',
{
'name': 'test',
# 'assertions': '%d' % (0,),
'time': '%0.3f' % (result.duration / 1000.0, ),
'classname': classname,
# 'status': '???',
})
if skipped:
try:
mime, content = result.artifacts['check']
except KeyError:
msg = ''
else:
msg = '\n'.join(['%s' % (c, ) for c in content])
xml.startElement('skipped', {
'message': msg,
})
xml.endElement('skipped')
elif errors:
xml.startElement('error', {
'type': 'TestError',
'message': '%s' % (result.result, ),
})
xml.endElement('error')
elif failures:
msg = TestCodes.MESSAGE.get(result.reason, '')
xml.startElement('failure', {
'type': 'TestFailure',
'message': msg,
})
xml.endElement('failure')
try:
mime, content = result.artifacts['stdout']
except KeyError:
pass
else:
xml.startElement('system-out', {})
xml.characters(self.utf8(content))
xml.endElement('system-out')
try:
mime, content = result.artifacts['stderr']
except KeyError:
pass
else:
xml.startElement('system-err', {})
xml.characters(self.utf8(content))
xml.endElement('system-err')
xml.endElement('testcase')
xml.endElement('testsuite')
xml.endDocument()
finally:
f_report.close()
super(Junit, self).end_test(result)
class WebDBXMLVisitor(NodeVisitor):
def __init__(self, document, builder):
NodeVisitor.__init__(self, document)
self.builder = builder
self._output = StringIO()
self.generator = XMLGenerator(self._output, "utf-8")
self.generator.outf = self._output
self.listenv = None
self.within_index = False
self.licount = 0
self.admonition = None
def find_figunumber(self, fig_id):
for docname, v in self.builder.env.toc_fignumbers.items():
if 'figure' not in v:
return None
if fig_id in v['figure']:
return v['figure'][fig_id]
return None
def newline(self):
self.generator.outf.write("\n")
def astext(self):
return self._output.getvalue()
def visit_document(self, node):
self.generator.startDocument()
self.generator.outf.write(
'<doc xmlns:aid="http://ns.adobe.com/AdobeInDesign/4.0/">')
self.sec_level = 0
def depart_document(self, node):
self.generator.endDocument()
self.generator.outf.write("</doc>")
def visit_paragraph(self, node):
# does not print <p> in block_quote
if isinstance(node.parent, nodes.block_quote):
return
# does not print <p> in list
if not self.listenv:
style = u'本文'
if self.admonition == 'column':
style = u'コラム本文'
self.generator.startElement('p', {'aid:pstyle': style})
self.generator.outf.write(" ") # insert empty zenkaku
def depart_paragraph(self, node):
if isinstance(node.parent, nodes.block_quote):
return
if not self.listenv:
self.generator.endElement('p')
self.newline()
def visit_section(self, node):
assert not self.within_index
self.sec_level += 1
def depart_section(self, node):
self.sec_level -= 1
def visit_title(self, node):
if self.admonition:
return
level = ""
if self.sec_level == 1:
level = u"大見出し"
elif self.sec_level == 2:
level = u"中見出し"
elif self.sec_level == 3:
level = u"小見出し"
self.generator.startElement('title', {"aid:pstyle": level})
def depart_title(self, node):
if self.admonition:
return
self.generator.endElement('title')
dtp = u'<?dtp level="{0}" section="{1}"?>'.format(
self.sec_level + 1,
node.astext(),
)
self.generator.outf.write(dtp)
self.newline()
def visit_Text(self, node):
self.generator.characters(node.astext())
def depart_Text(self, node):
pass
def visit_index(self, node):
# self.generator.startElement('index', {})
self.within_index = True
pass
def depart_index(self, node):
# self.generator.endElement('index')
self.within_index = False
pass
def visit_target(self, node):
# self.generator.startElement('section', {})
pass
def depart_target(self, node):
pass
def visit_emphasis(self, node):
self.generator.startElement("i", {'aid:cstyle': u"イタリック"})
def depart_emphasis(self, node):
self.generator.endElement("i")
def visit_math(self, node):
pass
def depart_math(self, node):
pass
def visit_doctest_block(self, node):
self.generator.startElement("programlisting",
{"language": "python"})
def depart_doctest_block(self, node):
self.generator.endElement("programlisting")
def visit_strong(self, node):
self.generator.startElement("b", {'aid:cstyle': u"太字"})
def depart_strong(self, node):
self.generator.endElement("b")
def visit_title_reference(self, node):
pass
def depart_title_reference(self, node):
pass
def visit_displaymath(self, node):
pass
def depart_displaymath(self, node):
pass
def visit_note(self, node):
self.generator.startElement('note', {})
def depart_note(self, node):
self.generator.endElement('note')
def visit_tip(self, node):
self.generator.startElement('tip', {})
def depart_tip(self, node):
self.generator.endElement('tip')
def visit_warning(self, node):
self.generator.startElement('warning', {})
def depart_warning(self, node):
self.generator.endElement('warning')
def visit_unknown_visit(self, node):
pass
def depart_unknown_visit(self, node):
pass
def visit_number_reference(self, node):
self.generator.startElement("b", {'aid:cstyle': u"太字"})
newtext = node.astext()
newtext = newtext.replace(" ", "").strip()
node.clear()
node.append(Text(newtext))
def depart_number_reference(self, node):
self.generator.endElement('b')
def visit_literal_block(self, node):
self.generator.startElement('p', {'aid:pstyle': u'半行アキ'})
self.generator.endElement('p')
self.newline()
if "highlight_args" in node.attributes.keys():
self.generator.startElement(
"listinfo",
{'aid:pstyle': u"リスト",
"language": node.attributes["language"]})
self._lit_block_tag = "listinfo"
else:
self.generator.startElement("literal", {'aid:pstyle': u"コマンド"})
self._lit_block_tag = "literal"
def depart_literal_block(self, node):
self.generator.endElement(self._lit_block_tag)
del self._lit_block_tag
self.newline()
def visit_literal(self, node):
self.generator.startElement("literal", {'aid:cstyle': u"コマンド"})
def depart_literal(self, node):
self.generator.endElement("literal")
def visit_comment(self, node):
raise nodes.SkipNode
def depart_comment(self, node):
pass
def visit_compound(self, node):
pass
def depart_compound(self, node):
pass
def visit_compact_paragraph(self, node):
pass
def depart_compact_paragraph(self, node):
pass
def visit_figure(self, node):
self.generator.startElement('img', {})
def depart_figure(self, node):
if self.imgfilepath:
self.generator.startElement('p', {'aid:pstyle': u"赤字段落"})
self.generator.outf.write(str(self.imgfilepath))
self.generator.endElement('p')
self.imgfilepath = None
self.generator.endElement('img')
self.newline()
def visit_image(self, node):
if 'uri' in node:
self.imgfilepath = ("file:///" + node['uri']).encode('utf-8')
def depart_image(self, node):
pass
def visit_caption(self, node):
self.generator.startElement('caption', {'aid:pstyle': u"キャプション"})
if isinstance(node.parent, nodes.figure):
figure_id = node.parent['ids'][0]
numbers = self.find_figunumber(figure_id)
if numbers:
self.generator.startElement('b', {'aid:cstyle': u"太字"})
s = "●図{0}".format('.'.join(map(str, numbers)))
self.generator.outf.write(s)
self.generator.endElement('b')
def depart_caption(self, node):
self.generator.endElement('caption')
self.newline()
def visit_bullet_list(self, node):
self.generator.startElement('p', {'aid:pstyle': u'半行アキ'})
self.generator.endElement('p')
self.newline()
self.listenv = "ul"
self.generator.startElement("ul", {})
def depart_bullet_list(self, node):
self.listenv = None
self.licount = 0
self.generator.endElement("ul")
self.newline()
def visit_enumerated_list(self, node):
self.generator.startElement('p', {'aid:pstyle': u'半行アキ'})
self.generator.endElement('p')
self.newline()
self.listenv = "ol"
self.generator.startElement("ol", {})
def depart_enumerated_list(self, node):
self.listenv = None
self.licount = 0
self.generator.endElement("ol")
self.newline()
def visit_reference(self, node):
pass
def depart_reference(self, node):
if 'refuri' in node:
self.generator.startElement("footnote", {})
self.generator.outf.write(node['refuri'])
self.generator.endElement("footnote")
# if node.get('secnumber'):
# self.body.append(('%s' + self.secnumber_suffix) %
# '.'.join(map(str, node['secnumber'])))
def visit_footnote(self, node):
self.generator.startElement("footnote", {})
def depart_footnote(self, node):
self.generator.endElement("footnote")
self.newline()
def visit_list_item(self, node):
if self.licount > 0:
self.newline()
self.licount += 1
self.generator.startElement("li", {"aid:pstyle": u"箇条書き"})
if self.listenv == "ul":
self.generator.outf.write("・")
else:
self.generator.startElement("span", {"aid:cstyle": u"丸文字"})
m = marumoji[self.licount]
self.generator.outf.write(m)
self.generator.endElement("span")
def depart_list_item(self, node):
self.generator.endElement("li")
def visit_field_list(self, node):
self.generator.startElement("variablelist", {})
def depart_field_list(self, node):
self.generator.endElement("variablelist")
def visit_field(self, node):
self.generator.startElement("varlistentry", {})
def depart_field(self, node):
self.generator.endElement("varlistentry")
def visit_field_name(self, node):
self.generator.startElement("term", {})
def depart_field_name(self, node):
self.generator.endElement("term")
def visit_field_body(self, node):
self.generator.startElement("listitem", {})
def depart_field_body(self, node):
self.generator.endElement("listitem")
def visit_definition_list(self, node):
self.generator.startElement('p', {'aid:pstyle': u'半行アキ'})
self.generator.endElement('p')
self.newline()
self.listenv = "dl"
self.generator.startElement("dl", {})
def depart_definition_list(self, node):
self.listenv = None
self.licount = 0
self.generator.endElement("dl")
def visit_definition_list_item(self, node):
pass
def depart_definition_list_item(self, node):
pass
def visit_term(self, node):
self.newline()
self.generator.startElement("dt", {'aid:pstyle': u"箇条書き"})
self.generator.outf.write("・")
def depart_term(self, node):
self.generator.endElement("dt")
def visit_definition(self, node):
self.generator.startElement("dd", {'aid:pstyle': u"箇条書き説明"})
def depart_definition(self, node):
self.generator.endElement("dd")
def visit_block_quote(self, node):
self.generator.startElement('p', {'aid:pstyle': u'半行アキ'})
self.generator.endElement('p')
self.newline()
self.generator.startElement("quote", {'aid:pstyle': u"引用"})
def depart_block_quote(self, node):
self.generator.endElement("quote")
self.newline()
def visit_inline(self, node):
if 'xref' in node['classes'] or 'term' in node['classes']:
self.add_text('*')
def depart_inline(self, node):
if 'xref' in node['classes'] or 'term' in node['classes']:
self.add_text('*')
def visit_todo_node(self, node):
pass
def depart_todo_node(self, node):
pass
def visit_container(self, node):
pass
def depart_container(self, node):
pass
def visit_admonition(self, node):
self.generator.startElement("title", {'aid:pstyle': u"コラム見出し"})
self.admonition = 'column'
self.newline()
def depart_admonition(self, node):
self.generator.endElement("title")
self.admonition = None
class UniversalXmlItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
self._configure(kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.encoding = self.encoding.upper()
self.indent = settings.get('UXML_INDENT', 0)
self.item_element = settings.get("UXML_ITEM", "item")
self.root_element = settings.get("UXML_ROOT", "items")
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
self._xg_characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self._beautify_newline(new_item=True)
def export_item(self, item):
self._beautify_indent(depth=1)
_attrs = {}
for key in item.keys():
if key.startswith("_"):
_attrs[key[1:]] = item[key]
_tag = self.item_element
if "tag" in _attrs:
_tag = _attrs.pop("tag")
self.xg.startElement(_tag, _attrs)
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
if name.startswith("_"):
continue
_sub_attrs = {}
if hasattr(value, 'items'):
for key in value.keys():
if key.startswith("_"):
_sub_attrs[key[1:]] = value[key]
_sub_tag = name
if "tag" in _sub_attrs:
_sub_tag = _sub_attrs.pop("tag")
_out_value = None
if "value" in _sub_attrs:
_out_value = _sub_attrs.pop("value")
self._export_xml_field(_sub_tag,
value,
depth=2,
attrs=_sub_attrs,
out_value=_out_value)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
self._beautify_newline(new_item=True)
def finish_exporting(self):
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self,
name,
serialized_value,
depth,
attrs=None,
out_value=None):
self._beautify_indent(depth=depth)
if attrs is None:
attrs = {}
if out_value is not None:
serialized_value = out_value
if serialized_value is None:
self.xg.ignorableWhitespace("<%s/>" % name)
self._beautify_newline()
return
self.xg.startElement(name, attrs)
if hasattr(serialized_value, 'items'):
self._beautify_newline()
for subname, value in serialized_value.items():
if subname.startswith("_"):
continue
_attrs = {}
if hasattr(value, 'items'):
for key in value.keys():
if key.startswith("_"):
_attrs[key[1:]] = value[key]
_tag = subname
if "tag" in _attrs:
_tag = _attrs.pop("tag")
_out_value = None
if "value" in _attrs:
_out_value = _attrs.pop("value")
self._export_xml_field(_tag,
value,
depth=depth + 1,
attrs=_attrs,
out_value=_out_value)
self._beautify_indent(depth=depth)
elif is_listlike(serialized_value):
self._beautify_newline()
_is_dict_inside = True
for value in serialized_value:
if not hasattr(value, 'items'):
_is_dict_inside = False
break
if _is_dict_inside:
for value in serialized_value:
_sub_attrs = {}
for key in value.keys():
if key.startswith("_"):
_sub_attrs[key[1:]] = value[key]
_sub_tag = 'value'
if "tag" in _sub_attrs:
_sub_tag = _sub_attrs.pop("tag")
_sub_out_value = None
if "value" in _sub_attrs:
_sub_out_value = _sub_attrs.pop("value")
self._export_xml_field(_sub_tag,
value,
depth=depth + 1,
out_value=_sub_out_value)
else:
for value in serialized_value:
self._export_xml_field('value', value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
self._xg_characters(serialized_value)
else:
self._xg_characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
# Workaround for https://bugs.python.org/issue17606
# Before Python 2.7.4 xml.sax.saxutils required bytes;
# since 2.7.4 it requires unicode. The bug is likely to be
# fixed in 2.7.6, but 2.7.6 will still support unicode,
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
if not isinstance(serialized_value, six.text_type):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else: # pragma: no cover
def _xg_characters(self, serialized_value):
return self.xg.characters(serialized_value)
class _XmlAnnotationWriter(_AnnotationWriter):
def __init__(self, file):
super().__init__(file, "1.0")
self.xmlgen = XMLGenerator(self.file, 'utf-8')
self._level = 0
def _indent(self, newline=True):
if newline:
self.xmlgen.ignorableWhitespace("\n")
self.xmlgen.ignorableWhitespace(" " * self._level)
def _add_version(self):
self._indent()
self.xmlgen.startElement("version", {})
self.xmlgen.characters(self.version)
self.xmlgen.endElement("version")
def open_root(self):
self.xmlgen.startDocument()
self.xmlgen.startElement("annotations", {})
self._level += 1
self._add_version()
def _add_meta(self, meta):
self._level += 1
for k, v in meta.items():
if isinstance(v, OrderedDict):
self._indent()
self.xmlgen.startElement(k, {})
self._add_meta(v)
self._indent()
self.xmlgen.endElement(k)
else:
self._indent()
self.xmlgen.startElement(k, {})
self.xmlgen.characters(v)
self.xmlgen.endElement(k)
self._level -= 1
def add_meta(self, meta):
self._indent()
self.xmlgen.startElement("meta", {})
self._add_meta(meta)
self._indent()
self.xmlgen.endElement("meta")
def open_track(self, track):
self._indent()
self.xmlgen.startElement("track", track)
self._level += 1
def open_image(self, image):
self._indent()
self.xmlgen.startElement("image", image)
self._level += 1
def open_box(self, box):
self._indent()
self.xmlgen.startElement("box", box)
self._level += 1
def add_attribute(self, attribute):
self._indent()
self.xmlgen.startElement("attribute", {"name": attribute["name"]})
self.xmlgen.characters(attribute["value"])
self.xmlgen.endElement("attribute")
def close_box(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("box")
def close_image(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("image")
def close_track(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("track")
def close_root(self):
self._level -= 1
self._indent()
self.xmlgen.endElement("annotations")
self.xmlgen.endDocument()
def end(self, node, digester):
'''
@see: Rule.end
'''
assert isinstance(digester, DigesterXMLUpdate), 'Invalid digester %s' % digester
XMLGenerator.endElement(digester, node.name)
class SeqXmlWriter(SequentialSequenceWriter):
"""Writes SeqRecords into seqXML file.
SeqXML requires the sequence alphabet be explicitly RNA, DNA or protein,
i.e. an instance or subclass of Bio.Alphapet.RNAAlphabet,
Bio.Alphapet.DNAAlphabet or Bio.Alphapet.ProteinAlphabet.
"""
def __init__(self,
handle,
source=None,
source_version=None,
species=None,
ncbiTaxId=None):
"""Create Object and start the xml generator."""
SequentialSequenceWriter.__init__(self, handle)
self.xml_generator = XMLGenerator(handle, "utf-8")
self.xml_generator.startDocument()
self.source = source
self.source_version = source_version
self.species = species
self.ncbiTaxId = ncbiTaxId
def write_header(self):
"""Write root node with document metadata."""
SequentialSequenceWriter.write_header(self)
attrs = {
"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
"xsi:noNamespaceSchemaLocation":
"http://www.seqxml.org/0.4/seqxml.xsd",
"seqXMLversion": "0.4"
}
if self.source is not None:
attrs["source"] = self.source
if self.source_version is not None:
attrs["sourceVersion"] = self.source_ersion
if self.species is not None:
if not isinstance(species, basestring):
raise TypeError("species should be of type string")
attrs["speciesName"] = self.species
if self.ncbiTaxId is not None:
if not isinstance(self.ncbiTaxId, (basestring, int)):
raise TypeError("ncbiTaxID should be of type string or int")
attrs["ncbiTaxID"] = self.ncbiTaxId
self.xml_generator.startElement("seqXML", AttributesImpl(attrs))
def write_record(self, record):
"""Write one record."""
if not record.id or record.id == "<unknown id>":
raise ValueError("SeqXML requires identifier")
if not isinstance(record.id, basestring):
raise TypeError("Identifier should be of type string")
attrb = {"id": record.id}
if "source" in record.annotations and self.source != record.annotations[
"source"]:
if not isinstance(record.annotations["source"], basestring):
raise TypeError("source should be of type string")
attrb["source"] = record.annotations["source"]
self.xml_generator.startElement("entry", AttributesImpl(attrb))
self._write_species(record)
self._write_description(record)
self._write_seq(record)
self._write_dbxrefs(record)
self._write_properties(record)
self.xml_generator.endElement("entry")
def write_footer(self):
"""Close the root node and finish the XML document."""
SequentialSequenceWriter.write_footer(self)
self.xml_generator.endElement("seqXML")
self.xml_generator.endDocument()
def _write_species(self, record):
"""Write the species if given."""
if "organism" in record.annotations and "ncbi_taxid" in record.annotations:
if not isinstance(record.annotations["organism"], basestring):
raise TypeError("organism should be of type string")
if not isinstance(record.annotations["ncbi_taxid"],
(basestring, int)):
raise TypeError("ncbiTaxID should be of type string or int")
#The local species definition is only written if it differs from the global species definition
if record.annotations[
"organism"] != self.species or record.annotations[
"ncbi_taxid"] != self.ncbiTaxId:
attr = {
"name": record.annotations["organism"],
"ncbiTaxID": record.annotations["ncbi_taxid"]
}
self.xml_generator.startElement("species",
AttributesImpl(attr))
self.xml_generator.endElement("species")
def _write_description(self, record):
"""Write the description if given."""
if record.description:
if not isinstance(record.description, basestring):
raise TypeError("Description should be of type string")
description = record.description
if description == "<unknown description>":
description = ""
if len(record.description) > 0:
self.xml_generator.startElement("description",
AttributesImpl({}))
self.xml_generator.characters(description)
self.xml_generator.endElement("description")
def _write_seq(self, record):
"""Write the sequence.
Note that SeqXML requires a DNA, RNA or protein alphabet.
"""
if isinstance(record.seq, UnknownSeq):
raise TypeError(
"Sequence type is UnknownSeq but SeqXML requires sequence")
seq = str(record.seq)
if not len(seq) > 0:
raise ValueError("The sequence length should be greater than 0")
#Get the base alphabet (underneath any Gapped or StopCodon encoding)
alpha = Alphabet._get_base_alphabet(record.seq.alphabet)
if isinstance(alpha, Alphabet.RNAAlphabet):
seqElem = "RNAseq"
elif isinstance(alpha, Alphabet.DNAAlphabet):
seqElem = "DNAseq"
elif isinstance(alpha, Alphabet.ProteinAlphabet):
seqElem = "AAseq"
else:
raise ValueError("Need a DNA, RNA or Protein alphabet")
self.xml_generator.startElement(seqElem, AttributesImpl({}))
self.xml_generator.characters(seq)
self.xml_generator.endElement(seqElem)
def _write_dbxrefs(self, record):
"""Write all database cross references."""
if record.dbxrefs is not None:
for dbxref in record.dbxrefs:
if not isinstance(dbxref, basestring):
raise TypeError("dbxrefs should be of type list of string")
if dbxref.find(':') < 1:
raise ValueError(
"dbxrefs should be in the form ['source:id', 'source:id' ]"
)
dbsource, dbid = dbxref.split(':', 1)
attr = {"source": dbsource, "id": dbid}
self.xml_generator.startElement("DBRef", AttributesImpl(attr))
self.xml_generator.endElement("DBRef")
def _write_properties(self, record):
"""Write all annotations that are key value pairs with values of a primitive type or list of primitive types."""
for key, value in record.annotations.items():
if key not in ("organism", "ncbi_taxid", "source"):
if value is None:
attr = {"name": key}
self.xml_generator.startElement("property",
AttributesImpl(attr))
self.xml_generator.endElement("property")
elif isinstance(value, list):
for v in value:
if isinstance(value, (int, float, basestring)):
attr = {"name": key, "value": v}
self.xml_generator.startElement(
"property", AttributesImpl(attr))
self.xml_generator.endElement("property")
elif isinstance(value, (int, float, basestring)):
attr = {"name": key, "value": str(value)}
self.xml_generator.startElement("property",
AttributesImpl(attr))
self.xml_generator.endElement("property")
class Logger:
def __init__(self):
self.logfile = os.environ.get("LOGFILE", "/dev/null")
self.logfile_handle = open(self.logfile, "wb")
self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
self.queue = queue.Queue(1000)
self.xml.startDocument()
self.xml.startElement("logfile", attrs={})
def close(self):
self.xml.endElement("logfile")
self.xml.endDocument()
self.logfile_handle.close()
def sanitise(self, message):
return "".join(ch for ch in message
if unicodedata.category(ch)[0] != "C")
def maybe_prefix(self, message, attributes):
if "machine" in attributes:
return "{}: {}".format(attributes["machine"], message)
return message
def log_line(self, message, attributes):
self.xml.startElement("line", attributes)
self.xml.characters(message)
self.xml.endElement("line")
def log(self, message, attributes={}):
eprint(self.maybe_prefix(message, attributes))
self.drain_log_queue()
self.log_line(message, attributes)
def enqueue(self, message):
self.queue.put(message)
def drain_log_queue(self):
try:
while True:
item = self.queue.get_nowait()
attributes = {"machine": item["machine"], "type": "serial"}
self.log_line(self.sanitise(item["msg"]), attributes)
except queue.Empty:
pass
@contextmanager
def nested(self, message, attributes={}):
eprint(self.maybe_prefix(message, attributes))
self.xml.startElement("nest", attrs={})
self.xml.startElement("head", attributes)
self.xml.characters(message)
self.xml.endElement("head")
tic = time.time()
self.drain_log_queue()
yield
self.drain_log_queue()
toc = time.time()
self.log("({:.2f} seconds)".format(toc - tic))
self.xml.endElement("nest")
class XmlAnnotationWriter:
VERSION = '1.1'
def __init__(self, f):
self.xmlgen = XMLGenerator(f, 'utf-8')
self._level = 0
def _indent(self, newline=True):
if newline:
self.xmlgen.ignorableWhitespace('\n')
self.xmlgen.ignorableWhitespace(' ' * self._level)
def _add_version(self):
self._indent()
self.xmlgen.startElement('version', {})
self.xmlgen.characters(self.VERSION)
self.xmlgen.endElement('version')
def open_root(self):
self.xmlgen.startDocument()
self.xmlgen.startElement('annotations', {})
self._level += 1
self._add_version()
def _add_meta(self, meta):
self._level += 1
for k, v in meta.items():
if isinstance(v, OrderedDict):
self._indent()
self.xmlgen.startElement(k, {})
self._add_meta(v)
self._indent()
self.xmlgen.endElement(k)
elif isinstance(v, list):
self._indent()
self.xmlgen.startElement(k, {})
for tup in v:
self._add_meta(OrderedDict([tup]))
self._indent()
self.xmlgen.endElement(k)
else:
self._indent()
self.xmlgen.startElement(k, {})
self.xmlgen.characters(v)
self.xmlgen.endElement(k)
self._level -= 1
def write_meta(self, meta):
self._indent()
self.xmlgen.startElement('meta', {})
self._add_meta(meta)
self._indent()
self.xmlgen.endElement('meta')
def open_track(self, track):
self._indent()
self.xmlgen.startElement('track', track)
self._level += 1
def open_image(self, image):
self._indent()
self.xmlgen.startElement('image', image)
self._level += 1
def open_box(self, box):
self._indent()
self.xmlgen.startElement('box', box)
self._level += 1
def open_polygon(self, polygon):
self._indent()
self.xmlgen.startElement('polygon', polygon)
self._level += 1
def open_polyline(self, polyline):
self._indent()
self.xmlgen.startElement('polyline', polyline)
self._level += 1
def open_points(self, points):
self._indent()
self.xmlgen.startElement('points', points)
self._level += 1
def open_tag(self, tag):
self._indent()
self.xmlgen.startElement("tag", tag)
self._level += 1
def add_attribute(self, attribute):
self._indent()
self.xmlgen.startElement('attribute', {'name': attribute['name']})
self.xmlgen.characters(attribute['value'])
self.xmlgen.endElement('attribute')
def _close_element(self, element):
self._level -= 1
self._indent()
self.xmlgen.endElement(element)
def close_box(self):
self._close_element('box')
def close_polygon(self):
self._close_element('polygon')
def close_polyline(self):
self._close_element('polyline')
def close_points(self):
self._close_element('points')
def close_tag(self):
self._close_element('tag')
def close_image(self):
self._close_element('image')
def close_track(self):
self._close_element('track')
def close_root(self):
self._close_element('annotations')
self.xmlgen.endDocument()
def _processEnd(self, nodes, name):
'''
@see: Digester._processEnd
'''
if not Digester._processEnd(self, nodes, name):
XMLGenerator.endElement(self, name)
class CombineSVG(object):
"""
CombineSVG combines multiple SVGa into a single SVG `output` stream.
Different `layer_name`s are added into a separate SVG layers (`<g>`).
Empty layers are stripped.
Note: Only the simplest SVGs with <path> elements are supported (e.g.
MapServer Cairo SVG output format).
"""
def __init__(self, output, root_id=None):
self.first = True
self.root_id = root_id
self.out = XMLGenerator(output, "utf-8")
self.out.startPrefixMapping(None, SVG_NS)
self.out.startDocument()
self.groups = []
self.actual_groups = []
def push_group(self, name):
"""
push_group adds a new SVG layer name to be used for the next SVG sub-document with
add.
"""
self.groups.append(name)
def pop_group(self):
"""
pop_group removes the last added SVG layer.
"""
self.groups = self.groups[:-1]
def balance_group(self):
"""
balance_group opens/closes <g> tags as necessary.
We can't create the <g> tags in push/pop_group as we do not want them for empty
layers.
"""
for want, got in zip_longest(self.groups, self.actual_groups):
if want == got:
continue
if got is None:
self.out.startElement("g", {"id": want})
self.out.characters("\n")
self.actual_groups.append(want)
self.balance_group()
return
if want is None or want != got:
self.out.endElement("g")
self.out.characters("\n")
self.actual_groups.pop()
self.balance_group()
return
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
def add(self, r):
"""
Add an SVG layer with `layer_name` from file object `r`.
"""
for evt, elem in iterparse(r, events=("start", "end")):
if evt == "start" and self.first and elem.tag == SVG_TAG:
# copy <svg> from first doc, otherwise ignore
svg_attrib = {(None, k): v for k, v in elem.attrib.items()}
if self.root_id:
svg_attrib[(None, "id")] = self.root_id
self.out.startElementNS((SVG_NS, "svg"), "svg", svg_attrib)
self.out.characters("\n")
continue
if evt == "end" and self.first and elem.tag == RECT_TAG:
# copy only first <rect>, used by Mapserver to draw blank canvas, but we
# need all additional to be transparent.
self.balance_group()
self.out.startElement("rect", elem.attrib)
self.out.endElement("rect")
self.out.characters("\n")
elif evt == "end" and elem.tag == PATH_TAG:
# all drawings are included in <path> tags
self.balance_group()
self.out.startElement("path", elem.attrib)
self.out.endElement("path")
self.out.characters("\n")
self.first = False
def close(self):
"""
Close all open SVG XML elements.
"""
self.groups = []
self.balance_group()
self.out.endElementNS((SVG_NS, "svg"), "svg")
self.out.endDocument()
class XmlItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
self.item_element = kwargs.pop('item_element', 'item')
self.root_element = kwargs.pop('root_element', 'items')
self._configure(kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
self._xg_characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self._beautify_newline(new_item=True)
def export_item(self, item):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
self._beautify_newline(new_item=True)
def finish_exporting(self):
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self, name, serialized_value, depth):
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
if hasattr(serialized_value, 'items'):
self._beautify_newline()
for subname, value in serialized_value.items():
self._export_xml_field(subname, value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif is_listlike(serialized_value):
self._beautify_newline()
for value in serialized_value:
self._export_xml_field('value', value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
self._xg_characters(serialized_value)
else:
self._xg_characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
# Workaround for https://bugs.python.org/issue17606
# Before Python 2.7.4 xml.sax.saxutils required bytes;
# since 2.7.4 it requires unicode. The bug is likely to be
# fixed in 2.7.6, but 2.7.6 will still support unicode,
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
if not isinstance(serialized_value, six.text_type):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else: # pragma: no cover
def _xg_characters(self, serialized_value):
return self.xg.characters(serialized_value)
class Logger:
def __init__(self) -> None:
self.logfile = os.environ.get("LOGFILE", "/dev/null")
self.logfile_handle = codecs.open(self.logfile, "wb")
self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
self.queue: "Queue[Dict[str, str]]" = Queue()
self.xml.startDocument()
self.xml.startElement("logfile", attrs={})
self._print_serial_logs = True
def close(self) -> None:
self.xml.endElement("logfile")
self.xml.endDocument()
self.logfile_handle.close()
def sanitise(self, message: str) -> str:
return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str:
if "machine" in attributes:
return "{}: {}".format(attributes["machine"], message)
return message
def log_line(self, message: str, attributes: Dict[str, str]) -> None:
self.xml.startElement("line", attributes)
self.xml.characters(message)
self.xml.endElement("line")
def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
eprint(self.maybe_prefix(message, attributes))
self.drain_log_queue()
self.log_line(message, attributes)
def log_serial(self, message: str, machine: str) -> None:
self.enqueue({"msg": message, "machine": machine, "type": "serial"})
if self._print_serial_logs:
eprint(Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL)
def enqueue(self, item: Dict[str, str]) -> None:
self.queue.put(item)
def drain_log_queue(self) -> None:
try:
while True:
item = self.queue.get_nowait()
msg = self.sanitise(item["msg"])
del item["msg"]
self.log_line(msg, item)
except Empty:
pass
@contextmanager
def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
eprint(self.maybe_prefix(message, attributes))
self.xml.startElement("nest", attrs={})
self.xml.startElement("head", attributes)
self.xml.characters(message)
self.xml.endElement("head")
tic = time.time()
self.drain_log_queue()
yield
self.drain_log_queue()
toc = time.time()
self.log("({:.2f} seconds)".format(toc - tic))
self.xml.endElement("nest")
class OSMWriter(object):
def __init__(self, filename=None, fp=None, compact_formatting=False):
if filename:
self.filename = filename
self.fp = open(self.filename, 'wb')
elif fp:
self.fp = fp
if sys.version_info < (3, 2):
if compact_formatting:
# xml.sax.saxutils.XMLGenerator only supports outputting
# compact empty tags since 3.2.
raise ValueError('compact formatting requires Python 3.2+')
else:
self.xmlfile = XMLGenerator(self.fp, 'utf-8')
else:
self.xmlfile = XMLGenerator(
self.fp, 'utf-8', short_empty_elements=compact_formatting)
self.xmlfile.startDocument()
# TODO include version
self.xmlfile.startElement("osm", {
'version': '0.6',
'generator': 'osmwriter'
})
def close(self, close_file=True):
self.xmlfile.characters("\n")
self.xmlfile.endElement("osm")
self.xmlfile.endDocument()
if close_file:
self.fp.close()
def node(self, id, lat, lon, tags=None, **metadata):
tags = tags or {}
self.xmlfile.characters("\n ")
attrs = {'id': t(id), 'lat': t(lat), 'lon': t(lon)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("node", attrs)
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("node")
def way(self, id, tags, nodeids, **metadata):
tags = tags or {}
nodeids = nodeids or []
self.xmlfile.characters("\n ")
attrs = {'id': t(id)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("way", attrs)
for nodeid in nodeids:
self.xmlfile.characters("\n ")
self.xmlfile.startElement("nd", {'ref': t(nodeid)})
self.xmlfile.endElement("nd")
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("way")
def relation(self, id, tags, members, **metadata):
tags = tags or {}
self.xmlfile.characters("\n ")
attrs = {'id': t(id)}
for key, value in metadata.items():
attrs[key] = t(value)
self.xmlfile.startElement("relation", attrs)
for member in members:
attrs = {}
if len(member) == 2:
type, ref = member
attrs = {'type': t(type), 'ref': t(ref)}
else:
type, ref, role = member
attrs = {'type': t(type), 'ref': t(ref), 'role': t(role)}
self.xmlfile.characters("\n ")
self.xmlfile.startElement("member", attrs)
self.xmlfile.endElement("member")
for key, value in tags.items():
self.xmlfile.characters("\n ")
self.xmlfile.startElement("tag", {'k': key, 'v': value})
self.xmlfile.endElement("tag")
self.xmlfile.characters("\n ")
self.xmlfile.endElement("relation")
def dump_xml(self, output_dir):
"""
Dumps test result to xml
"""
self.buffer = False
with open(os.path.join(output_dir, 'junit.xml'), 'w') as output:
document = XMLGenerator(output, 'utf-8')
document.startDocument()
document.startElement('testsuites', AttributesImpl({}))
suites = groupby(self.testInfos,
key=lambda test_info: self.test_case_name(
test_info.test_method))
for suite_name, suite in suites:
document.startElement('testsuite',
AttributesImpl({'name': suite_name}))
for test_info in suite:
document.startElement(
'testcase',
AttributesImpl({
'classname':
suite_name,
'name':
self.test_method_name(test_info.test_method),
'time':
'%3f' % total_seconds(test_info.end_time -
test_info.start_time)
}))
if test_info.result == TestInfo.RESULT.ERROR:
document.startElement(
'error',
AttributesImpl(
{'message': unicode(test_info.err[1])}))
document.characters(
self._exc_info_to_string(test_info.err,
test_info.test_method))
document.endElement('error')
elif test_info.result == TestInfo.RESULT.FAILURE:
document.startElement(
'failure',
AttributesImpl(
{'message': unicode(test_info.err[1])}))
document.characters(
self._exc_info_to_string(test_info.err,
test_info.test_method))
document.endElement('failure')
elif test_info.result == TestInfo.RESULT.UNEXPECTED_SUCCESS:
document.startElement(
'error',
AttributesImpl({'message': 'Unexpected success'}))
document.endElement('error')
elif test_info.result == TestInfo.RESULT.SKIPPED:
document.startElement('skipped', AttributesImpl({}))
document.characters(test_info.reason)
document.endElement('skipped')
if test_info.stdout:
document.startElement('system-out', AttributesImpl({}))
document.characters(test_info.stdout)
document.endElement('system-out')
if test_info.stderr:
document.startElement('system-err', AttributesImpl({}))
document.characters(test_info.stderr)
document.endElement('system-err')
document.endElement('testcase')
document.endElement('testsuite')
document.endElement('testsuites')
document.endDocument()
def param_to_xml(params, encoding='iso-8859-1'):
from xml.sax.saxutils import XMLGenerator
from StringIO import StringIO
output = StringIO()
xml = XMLGenerator(output, encoding)
xml.startDocument()
xml.startElement(
'TS_params', {
'cutoff': ' '.join([str(x) for x in params['rcut']]),
'n_types': str(params['nspecies']),
'betapol': str(params['betapol']),
'maxipol': str(params['maxipol']),
'tolpol': str(params['tolpol']),
'pred_order': str(params['pred_order']),
'yukalpha': str(params['yukalpha']),
'yuksmoothlength': str(params['yuksmoothlength']),
'tewald': params['tewald'] and 'T' or 'F',
'raggio': str(params['raggio']),
'a_ew': str(params['a_ew']),
'gcut': str(params['gcut']),
'iesr': ' '.join([str(x) for x in params.get('iesr', [0, 0, 0])])
})
ti_tj_to_index = {}
n = 0
for ti in range(params['nspecies']):
for tj in range(params['nspecies']):
if tj > ti: continue
ti_tj_to_index[(ti, tj)] = n
n += 1
for ti in range(params['nspecies']):
zi = atomic_number(params['species'][ti])
xml.startElement(
'per_type_data', {
'type': str(ti + 1),
'atomic_num': str(zi),
'pol': str(params['pol'][ti]),
'z': str(params['z'][ti])
})
xml.endElement('per_type_data')
for tj in range(params['nspecies']):
if tj > ti: continue
idx = ti_tj_to_index[(ti, tj)]
zj = atomic_number(params['species'][tj])
xml.startElement(
'per_pair_data', {
'atnum_i': str(zi),
'atnum_j': str(zj),
'D_ms': str(params['d_ms'][idx]),
'gamma_ms': str(params['gamma_ms'][idx]),
'R_ms': str(params['r_ms'][idx]),
'B_pol': str(params['bpol'][idx]),
'C_pol': str(params['cpol'][idx]),
})
xml.endElement('per_pair_data')
xml.endElement('TS_params')
xml.endDocument()
return output.getvalue()
