def present(name,
config=None,
**kwargs):
'''
Ensure the job is present in the Jenkins
configured jobs
name
The unique name for the Jenkins job
config
The Salt URL for the file to use for
configuring the job.
'''
test = __opts__['test']
ret = {'name': name,
'result': True,
'changes': {},
'comment': ['Job {0} is up to date.'.format(name)]}
if test:
status = 'CREATED'
ret['changes'][name] = status
ret['comment'] = 'Job %s %s' % (name, status.lower())
else:
_current_job_config = ''
_job_exists = True
try:
_current_job_config = __salt__['jenkins.get_job_config'](name)
except jenkins.NotFoundException:
_job_exists = False
if _job_exists:
buf = six.moves.StringIO(_current_job_config)
oldXMLstring = buf.read()
cached_source_path = __salt__['cp.cache_file'](config, __env__)
with salt.utils.fopen(cached_source_path) as _fp:
newXMLstring = _fp.read()
if not _elements_equal(oldXMLstring, newXMLstring):
oldXML = ET.fromstring(oldXMLstring)
newXML = ET.fromstring(newXMLstring)
diff = difflib.unified_diff(
ET.tostringlist(oldXML, encoding='utf8', method='xml'),
ET.tostringlist(newXML, encoding='utf8', method='xml'), lineterm='')
__salt__['jenkins.update_job'](name, config, __env__)
ret['changes'][name] = ''.join(diff)
ret['comment'].append('Job {0} updated.'.format(name))
else:
cached_source_path = __salt__['cp.cache_file'](config, __env__)
with salt.utils.fopen(cached_source_path) as _fp:
new_config_xml = _fp.read()
__salt__['jenkins.create_job'](name, config, __env__)
buf = six.moves.StringIO(new_config_xml)
diff = difflib.unified_diff('', buf.readlines(), lineterm='')
ret['changes'][name] = ''.join(diff)
ret['comment'].append('Job {0} added.'.format(name))
ret['comment'] = '\n'.join(ret['comment'])
return ret
def present(name, config=None, **kwargs):
'''
Ensure the job is present in the Jenkins configured jobs
name
The unique name for the Jenkins job
config
The Salt URL for the file to use for configuring the job
'''
ret = {
'name': name,
'result': True,
'changes': {},
'comment': ['Job {0} is up to date.'.format(name)]
}
if __salt__['jenkins.job_exists'](name):
_current_job_config = __salt__['jenkins.get_job_config'](name)
buf = six.moves.StringIO(_current_job_config)
oldXML = ET.fromstring(buf.read())
cached_source_path = __salt__['cp.cache_file'](config, __env__)
with salt.utils.files.fopen(cached_source_path) as _fp:
newXML = ET.fromstring(_fp.read())
if not _elements_equal(oldXML, newXML):
diff = difflib.unified_diff(ET.tostringlist(oldXML,
encoding='utf8',
method='xml'),
ET.tostringlist(newXML,
encoding='utf8',
method='xml'),
lineterm='')
try:
__salt__['jenkins.update_job'](name, config, __env__)
except CommandExecutionError as exc:
return _fail(ret, exc.strerror)
else:
ret['changes'] = ''.join(diff)
ret['comment'].append('Job \'{0}\' updated.'.format(name))
else:
cached_source_path = __salt__['cp.cache_file'](config, __env__)
with salt.utils.files.fopen(cached_source_path) as _fp:
new_config_xml = _fp.read()
try:
__salt__['jenkins.create_job'](name, config, __env__)
except CommandExecutionError as exc:
return _fail(ret, exc.strerror)
buf = six.moves.StringIO(new_config_xml)
diff = difflib.unified_diff('', buf.readlines(), lineterm='')
ret['changes'][name] = ''.join(diff)
ret['comment'].append('Job \'{0}\' added.'.format(name))
ret['comment'] = '\n'.join(ret['comment'])
return ret
def present(name, config=None, **kwargs):
"""
Ensure the job is present in the Jenkins configured jobs
name
The unique name for the Jenkins job
config
The Salt URL for the file to use for configuring the job
"""
ret = {
"name": name,
"result": True,
"changes": {},
"comment": ["Job {0} is up to date.".format(name)],
}
if __salt__["jenkins.job_exists"](name):
_current_job_config = __salt__["jenkins.get_job_config"](name)
buf = six.moves.StringIO(_current_job_config)
oldXML = ET.fromstring(buf.read())
cached_source_path = __salt__["cp.cache_file"](config, __env__)
with salt.utils.files.fopen(cached_source_path) as _fp:
newXML = ET.fromstring(
salt.utils.stringutils.to_unicode(_fp.read()))
if not _elements_equal(oldXML, newXML):
diff = difflib.unified_diff(
ET.tostringlist(oldXML, encoding="utf8", method="xml"),
ET.tostringlist(newXML, encoding="utf8", method="xml"),
lineterm="",
)
try:
__salt__["jenkins.update_job"](name, config, __env__)
except CommandExecutionError as exc:
return _fail(ret, exc.strerror)
else:
ret["changes"] = "".join(diff)
ret["comment"].append("Job '{0}' updated.".format(name))
else:
cached_source_path = __salt__["cp.cache_file"](config, __env__)
with salt.utils.files.fopen(cached_source_path) as _fp:
new_config_xml = salt.utils.stringutils.to_unicode(_fp.read())
try:
__salt__["jenkins.create_job"](name, config, __env__)
except CommandExecutionError as exc:
return _fail(ret, exc.strerror)
buf = six.moves.StringIO(new_config_xml)
diff = difflib.unified_diff("", buf.readlines(), lineterm="")
ret["changes"][name] = "".join(diff)
ret["comment"].append("Job '{0}' added.".format(name))
ret["comment"] = "\n".join(ret["comment"])
return ret
def _diff_configs(remote_config, new_config):
remote_root = ElementTree.fromstring(remote_config)
new_root = ElementTree.fromstring(new_config)
# ignore description which contains timestamp
remote_root.find('description').text = ''
new_root.find('description').text = ''
if ElementTree.tostring(remote_root) == ElementTree.tostring(new_root):
return []
lines1 = ElementTree.tostringlist(remote_root, encoding='unicode')
lines2 = ElementTree.tostringlist(new_root, encoding='unicode')
return difflib.unified_diff(
lines1, lines2, 'remote config', 'new config', n=0)
def generate_tag(tag, content, attributes=None):
"""Generate a complete html tag using the ElementTree module. tag and
content are strings, the attributes argument is a dictionary. As
a convenience, if the content is ' /', a self-closing tag is generated."""
content = six.text_type(content)
# In PY2, ElementTree tostringlist only works with bytes, not with
# unicode().
enc = 'unicode'
if six.PY2:
enc = 'UTF-8'
if not tag:
return content
element = ElementTree.Element(tag, attrib=attributes)
# FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried
# adding text by assigning it to element_tag.text. That results in
# non-ascii text being html-entity encoded. Not bad, but not entirely
# matching php-textile either.
element_tag = ElementTree.tostringlist(element,
encoding=enc,
method='html')
if six.PY2:
element_tag = [v.decode(enc) for v in element_tag]
element_tag.insert(len(element_tag) - 1, content)
element_text = ''.join(element_tag)
return element_text
def generate_tag(tag, content, attributes=None):
"""Generate a complete html tag using the ElementTree module. tag and
content are strings, the attributes argument is a dictionary. As
a convenience, if the content is ' /', a self-closing tag is generated."""
content = six.text_type(content)
enc = 'unicode'
if six.PY2:
enc = 'UTF-8'
if not tag:
return content
element = ElementTree.Element(tag, attrib=attributes)
# FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried
# adding text by assigning it to element_tag.text. That results in
# non-ascii text being html-entity encoded. Not bad, but not entirely
# matching php-textile either.
try:
element_tag = ElementTree.tostringlist(element, encoding=enc,
method='html')
element_tag.insert(len(element_tag) - 1, content)
element_text = ''.join(element_tag)
except AttributeError:
# Python 2.6 doesn't have the tostringlist method, so we have to treat
# it differently.
element_tag = ElementTree.tostring(element, encoding=enc)
element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n",
'', element_tag)
if content != six.text_type(' /'):
element_text = element_text.rstrip(' />')
element_text = six.text_type('{0}>{1}</{2}>').format(six.text_type(
element_text), content, tag)
return element_text
def _write_tree(self, tree, begin=None, end=None, indent=0):
"""
Write a part of the tree stringlist.
Begin and end are strings that mark begin and end of the part of
the tree that needs to be written. May not work if elementtrees
splitting behaviour gets really weird.
"""
self.xml_output_file.write(indent * ' ')
write = True if begin is None else False
for text in ElementTree.tostringlist(tree):
if begin is not None and begin in text:
write = True
if write:
self.xml_output_file.write(text)
if end is not None and end in text:
break
if (not text.endswith('\n')) and (end is not None):
self.xml_output_file.write('\n')
def xml_add_vcpupin_item(self, xml_file, num):
"""
<vcpu placement="static">3</vcpu>
<cputune>
<vcpupin cpuset="1" vcpu="0" />
<vcpupin cpuset="2" vcpu="1" />
<vcpupin cpuset="3" vcpu="2" />
</cputune>
"""
tree = ET.parse(xml_file)
root = tree.getroot()
vcpu_item = ET.ElementPath.find(root, "./vcpu")
current_num = 0
if None != vcpu_item:
current_num = int(vcpu_item.text)
vcpu_item.text = str(num)
item = ET.ElementPath.find(root, "./cputune")
sub_item = ET.ElementPath.find(root, "./cputune/vcpupin")
if num > current_num:
for i in range(num - current_num):
item.append(sub_item)
sub_item_list = list(item)
for i in range(sub_item_list.__len__()):
sub_item_list[i].set(str("vcpu"), str(i))
sub_item_list[i].set(str("cpuset"), str(i))
sub_item_list.sort()
print(ET.tostringlist(item))
tree.write(xml_file)
pass
def etd(self):
""" Given origin and direction, list train departure times. """
root = get_xml_tree_from_url(self.url)
try:
time = root.find("time").text
except AttributeError:
msg = "\n".join(ET.tostringlist(root, method="text"))
raise Exception(msg)
station = root.find("station/name").text
print("{} at {}".format(station, time))
for etd in root.findall("station/etd"):
dest = etd.find("destination").text.replace(" ", "")
for estimate in etd.findall("estimate"):
minutes = estimate.find("minutes").text
length = estimate.find("length").text
if minutes.lower() == "leaving":
minute_str = "now leaving"
depart_at = "--:--"
else:
minute_str = "in {:>2} minutes".format(minutes)
if minutes.strip() == "1":
minute_str = minute_str[:-1]
departure = datetime.strptime(time, "%H:%M:%S %p %Z")\
+ timedelta(minutes=int(minutes))
depart_at = datetime.strftime(departure, "%H:%M")
print("\t[{}] {:>2} car {} train {},"\
.format(depart_at, length, dest, minute_str))
def generate_tag(tag, content, attributes=None):
"""Generate a complete html tag using the ElementTree module. tag and
content are strings, the attributes argument is a dictionary. As
a convenience, if the content is ' /', a self-closing tag is generated."""
content = six.text_type(content)
enc = 'unicode'
if six.PY2:
enc = 'UTF-8'
if not tag:
return content
element = ElementTree.Element(tag, attrib=attributes)
# FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried
# adding text by assigning it to element_tag.text. That results in
# non-ascii text being html-entity encoded. Not bad, but not entirely
# matching php-textile either.
try:
element_tag = ElementTree.tostringlist(element,
encoding=enc,
method='html')
element_tag.insert(len(element_tag) - 1, content)
element_text = ''.join(element_tag)
except AttributeError:
# Python 2.6 doesn't have the tostringlist method, so we have to treat
# it differently.
element_tag = ElementTree.tostring(element, encoding=enc)
element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n",
'', element_tag)
if content != six.text_type(' /'):
element_text = element_text.rstrip(' />')
element_text = six.text_type('{0}>{1}</{2}>').format(
six.text_type(element_text), content, tag)
return element_text
def generate_tag(tag, content, attributes=None):
"""Generate a complete html tag using the ElementTree module. tag and
content are strings, the attributes argument is a dictionary. As
a convenience, if the content is ' /', a self-closing tag is generated."""
enc = 'unicode'
if not tag:
return content
element = ElementTree.Element(tag, attrib=attributes)
# Sort attributes for Python 3.8+, as suggested in
# https://docs.python.org/3/library/xml.etree.elementtree.html
if len(element.attrib) > 1:
# adjust attribute order, e.g. by sorting
attribs = sorted(element.attrib.items())
element.attrib.clear()
element.attrib.update(attribs)
# FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried
# adding text by assigning it to element_tag.text. That results in
# non-ascii text being html-entity encoded. Not bad, but not entirely
# matching php-textile either.
element_tag = ElementTree.tostringlist(element,
encoding=enc,
method='html')
element_tag.insert(len(element_tag) - 1, content)
element_text = ''.join(element_tag)
return element_text
def _write_tree(self, tree, begin=None, end=None, indent=0):
"""
Write a part of the tree stringlist.
Begin and end are strings that mark begin and end of the part of
the tree that needs to be written. May not work if elementtrees
splitting behaviour gets really weird.
"""
self.xml_output_file.write(indent * ' ')
write = True if begin is None else False
for text in ElementTree.tostringlist(tree):
if begin is not None and begin in text:
write = True
if write:
self.xml_output_file.write(text)
if end is not None and end in text:
break
if (not text.endswith('\n')) and (end is not None):
self.xml_output_file.write('\n')
def tostringlist(element, encoding=None, method=None):
"""
@summary: ElementTree.tostringlist wrapper that converts an Element tree
into a pretty printed list of strings
@see: ElementTree.tostring
"""
_prettyFormat(element, level=0)
return ET.tostringlist(element, encoding=encoding, method=method)
def test_output():
with open('tests/sample.txt') as f:
test_output = f.read()
actualXml = output2xml(test_output)
actual = ET.tostringlist(actualXml, encoding='unicode')
expectedXmlRoot = ET.parse('tests/expected.xml').getroot()
expected = ET.tostringlist(expectedXmlRoot, encoding='unicode')
# 改行を排除
regex = re.compile(r"^\s*$")
expected = [line for line in expected if not regex.match(line)]
assert len(actual) == len(expected)
for f, b in zip(actual, expected):
print(f, b)
assert f == b
def fine_format_xml(root):
lines = ET.tostringlist(root, encoding='unicode')
result = []
prev = '>'
for line in lines:
if (line.startswith('<') and prev.endswith('>')
) and not (line.startswith('</var>') and prev.endswith('</cond>')):
result.append('\n')
result.append(line)
prev = line
return ''.join(result)
def to_str_list(elementTree: ET.ElementTree) -> List[str]:
"""
Turn an ElementTree object into a list of str.
Args:
elementTree (ElementTree): A ElementTree object with all the data of the ossec.conf.
Returns:
(list of str): A list of str containing all the lines of the ossec.conf.
"""
return ET.tostringlist(elementTree.getroot(), encoding="unicode")
def makePackageDict(dir_path, Repos_dict):
package_dict = {}
komodo_dict = komodo.makeDataList('./komodo')
# Parse the files one by one
for f in makeManifestFileList(dir_path):
with open(f, 'r') as open_file:
tree = ET.parse(open_file)
root = tree.getroot()
name = root.find('name').text
repo = Repos_dict[f[f.rindex('/')+1:f.rindex('-')]]
package = Package(name, repo)
for person in extractPeople(root, 'author'):
package.authors.add(person)
for person in extractPeople(root, 'maintainer'):
package.maintainers.add(person)
if '<metapackage' in ET.tostringlist(root):
package.isMetapackage = True
# print " Found metapackage", name
predescription = extractTag(root, 'description')[0].strip().replace('\n','').replace('\r','').replace('\t','').replace('"','').replace(' ','')
package.description = re.sub(r'<.+?>', '', predescription)
package.licenses = set(extractTag(root, 'license'))
prewiki = extractTag(root, 'url')
if prewiki != None and None not in prewiki: prewiki = [url for url in prewiki if 'wiki'in url and 'git' not in url]
if len(prewiki) > 0: package.wiki = prewiki[0]
package.buildtool_depend = set(extractTag(root, 'buildtool_depend'))
package.build_depend = set(extractTag(root, 'build_depend'))
package.run_depend = set(extractTag(root, 'run_depend'))
if (package.name in komodo_dict):
package.runtime = komodo_dict[package.name]
package_dict[package.name] = package
# Calculate reverse dependencies for each package
# Although most are listed under the depends_on key there are pkgs missed
print " Verifying dependencies:",
counts = [0,0]
for p in package_dict.values():
deps = [p.buildtool_depend, p.build_depend, p.run_depend]
for dep in deps:
clone_depend = dep.copy() # Clone these deps to remove invalids when looping
for d in clone_depend:
if d not in package_dict:
dep.remove(d) # Remove deps that are not known packages
counts[0] += 1
else: counts[1] += 1
print " Invalid ", counts[0],
print " Valid ", counts[1]
return package_dict
def _serialize_xml(self, root):
"""It serializes the dynamic xml created and converts
it to a string. This is done before sending the
xml to the ILO.
:param root: root of the dynamic xml.
"""
if hasattr(etree, 'tostringlist'):
xml = '\r\n'.join(etree.tostringlist(root)) + '\r\n'
else:
xml = etree.tostring(root) + '\r\n'
return xml
def _serialize_xml(self, root):
"""It serializes the dynamic xml created and converts
it to a string. This is done before sending the
xml to the ILO.
:param root: root of the dynamic xml.
"""
if hasattr(etree, 'tostringlist'):
xml = '\r\n'.join(etree.tostringlist(root)) + '\r\n'
else:
xml = etree.tostring(root) + '\r\n'
return xml
def add_repository(
self,
id: str,
url: str,
):
repository = et.Element('repository')
et.SubElement('id', repository).text = id
et.SubElement('url', repository).text = url
self._repositories.append('')
for line in et.tostringlist(repository, encoding="unicode"):
self._repositories.append(line)
def do_pretty(self, fname):
""" pretty print the in-memry XML tree to output fname ("-" means stdout) """
lines = ET.tostringlist(self.root)
dom = xml.dom.minidom.parseString("".join(l for l in lines
if l and l.strip()))
pretty_xml = dom.toprettyxml(indent=" ",
encoding=self.xml_pi.get(
"encoding", None))
if fname == "-":
fprint(pretty_xml, end="")
else:
with open_for_writing(fname, "b") as fob:
fprint(pretty_xml, end="", file=fob)
def write_adaptor_declaration(self, interface_el, notifications,
request_responses, out):
global prefix_class_item
def glue_strings(strings):
ret = list()
curstr = ''
for str in strings:
curstr = curstr + str
if (str[-1] == '>'):
ret.append(curstr)
curstr = ''
return ret
ifacename = interface_el.get('name')
out.write("class " + ifacename +
"Adaptor : public QDBusAbstractAdaptor {\n")
out.write(" Q_OBJECT\n")
out.write(" Q_CLASSINFO(\"D-Bus Interface\", \"" +
self.interface_path + '.' + ifacename + "\")\n")
out.write(" Q_CLASSINFO(\"D-Bus Introspection\",\n")
introspection_el = self.create_introspection_iface_el(interface_el)
introspection = glue_strings(
ElementTree.tostringlist(introspection_el))
for str in introspection:
str = str.replace('"', '\\"')
out.write('"' + str + '"' + "\n")
out.write(" )\n")
out.write(" public:\n")
out.write(" explicit " + ifacename +
"Adaptor(QObject *parent = 0);\n")
out.write(" void SetApi(Q%sItem*);\n" % prefix_class_item)
out.write(" DBusController *dbusController;\n")
out.write(" public slots:\n")
for (request, response) in request_responses:
signature = self.make_method_signature(request, response,
ifacename, False)
out.write(" " + signature + ";\n")
out.write(" signals:\n")
for n in notifications:
signature = self.make_signal_signature(n, ifacename, True)
out.write(" " + signature + ";\n")
out.write(" private slots:\n")
for n in notifications:
signature = self.make_qml_signal_signature(n, ifacename,
n.get('name') + '_qml',
False)
out.write(" " + signature + ";\n")
out.write(" private:\n")
out.write(" Q%sItem* api_;\n" % prefix_class_item)
out.write("};\n\n")
def ToElementString(self, element):
if element.tag == 'Iterate':
outStr = "Iterate: " + element.attrib['XPath']
return outStr
strList = ElementTree.tostringlist(element)
outStr = ""
for s in strList:
outStr = outStr + " " + s.decode('Utf-8')
if s == '>':
break
return outStr
def add_dependency(
self,
artifact_id: str,
version: str = '${matsim.version}',
group_id: str = 'org.matsim.contrib',
):
dependency = et.Element('dependency')
et.SubElement('groupId', dependency).text = group_id
et.SubElement('artifactId', dependency).text = artifact_id
et.SubElement('version', dependency).text = version
self._dependencies.append('')
for line in et.tostringlist(dependency, encoding="unicode"):
self._dependencies.append(line)
def convert(self, outpath):
xroot = ET.Element('smses')
n = 0
for jconv in self.jfile['conversations']:
for jmsg in jconv:
xmsg = self.build_message(jmsg)
xroot.append(xmsg)
n += 1
xroot.attrib['count'] = str(n)
with open(str(outpath), 'wb') as fd:
for line in ET.tostringlist(xroot, method='xml'):
fd.write(line)
def _serialize_xml(self, root):
"""Serialize XML data into string
It serializes the dynamic xml created and converts
it to a string. This is done before sending the
xml to the ILO.
:param root: root of the dynamic xml.
"""
if hasattr(etree, 'tostringlist'):
if six.PY3:
xml_content_list = [
x.decode("utf-8") for x in etree.tostringlist(root)]
else:
xml_content_list = etree.tostringlist(root)
xml = '\r\n'.join(xml_content_list) + '\r\n'
else:
if six.PY3:
xml_content = etree.tostring(root).decode("utf-8")
else:
xml_content = etree.tostring(root)
xml = xml_content + '\r\n'
return xml
def process(path, style):
xml = ET.parse(path)
e = xml.getroot().find("n:head/n:style[@type='text/css']", NS)
if e is not None:
e.text="$$STYLE$$"
else:
print(f'WARNING: {path} has no style element')
for e in xml.getroot().findall(".//n:img", NS):
with open(path.parent / e.get('src'), 'rb') as f:
img = base64.b64encode(f.read()).decode()
e.set('src', 'data:image/png;base64,' + img)
xml_str = ET.tostringlist(xml.getroot(), encoding='unicode')
with open(path.with_suffix('.html'), 'w', encoding='utf-8') as f:
f.write(''.join(style if p == '$$STYLE$$' else p for p in xml_str))
def createLog(lines):
root = xml.Element('django-objects', attrib={'version' : '1.0'})
for l in lines:
child_line = xml.SubElement(root, 'object', attrib={'pk' : str(l.id), 'model': "log.line"})
line_message = xml.SubElement(child_line, 'field', attrib={'type' : "TextField", 'name' : "message"})
line_message.text = formatMsg(l.message)
line_nick = xml.SubElement(child_line, 'field', attrib={'type' : "CharField", 'name' : "nick"})
line_nick.text = formatMsg(l.nick)
line_hostname = xml.SubElement(child_line, 'field', attrib={'type' : "CharField", 'name' : "hostname"})
line_hostname.text = formatMsg(l.hostname)
line_channel = xml.SubElement(child_line, 'field', attrib={'type' : "CharField", 'name' : "channel"})
line_channel.text = formatMsg(l.channel)
line_timestamp = xml.SubElement(child_line, 'field', attrib={'type' : "DateTimeField", 'name' : "timestamp"})
line_timestamp.text = str(l.timestamp)
line_msgType = xml.SubElement(child_line, 'field', attrib={'type' : "CharField", 'name' : "msgType"})
line_msgType.text = formatMsg(l.msgType)
return xml.tostringlist(root, encoding="utf8", method='xml')
def tpyos(word_dir, docm_dir):
"""
build reference list of words
each line from all files in word_dir is considered a word
all words converted to lowercase, so not suitable to detect capitalization defects
encoding assumed: UTF-8
"""
ref_words = set()
for word_file in glob.glob(word_dir + os.sep + '*'):
if os.path.isfile(word_file):
with open(word_file, 'r', encoding='UTF-8') as wf:
ref_words.update(set(wf.read().lower().split('\n')))
"""
build typo dictionary
input document assumed as docx/epub, they are unzipped in temp dir
then each filename containing xml/htm is processed --> need to improve this
xhtml format files likely not processed correctly because of text formatting mixing
"""
tpyo_words = dict()
re_strip = re.compile(r'^[^a-zA-Z]*|[^a-zA-Z]*$')
re_valid_words = re.compile(r'[a-zA-Z]')
for doc_file in glob.glob(docm_dir + os.sep + '*'):
if re.search(r'\.(epub|docx)$', doc_file) and os.path.isfile(doc_file):
with zipfile.ZipFile(doc_file, 'r') as zp, tempfile.TemporaryDirectory() as td:
zp.extractall(td)
rp = os.sep + '**' + os.sep
all_files = glob.glob(td + rp + '*.*htm*', recursive=True)
all_files.extend(glob.glob(td + rp + '*.*xml*', recursive=True))
for ip_file in all_files:
xml_root = ET.parse(ip_file).getroot()
all_text = ET.tostringlist(xml_root, encoding='unicode', method='text')
for item in all_text:
for word in re.split(r'[\s—]', item):
word = re_strip.sub('', word)
# convert to ascii single quotes and lowercase the string for ref_words comparison
word_l = word.translate({0x2019:0x27}).lower()
if re_valid_words.search(word_l) and word_l not in ref_words:
if word in tpyo_words:
tpyo_words[word] = tpyo_words[word] + 1
else:
tpyo_words[word] = 1
return tpyo_words
def __init__(self,file=None):
cwd = os.getcwd()
self.my_string = None
if file is not None:
self.file = file
try:
f = tarfile.open(cwd + '/' + self.file,'r')
f.extractall() #Extract all file to the current working directory.
names = f.getnames()
corpus = []
for name in names:
if (name).endswith("xml") is True:
tree = ET.parse(cwd + '/'+ name)
txt = ET.tostringlist(tree.getroot(), encoding='utf-8', method='text')
corpus.extend(txt)
self.my_string = str(corpus).strip('[]')
except:
print('cannot open', self.file)
def process_output(fmt, data):
debug(type(data))
if re.match(r'^yam?l', str(fmt), re.I):
return (yaml.dump(data))
if re.match(r'^xml', str(fmt), re.I):
try:
return (ET.tostring(data.items()))
except AttributeError:
debug('return(ET.tostring(data.items())) didnt work')
try:
return (ET.tostringlist(data))
except AttributeError:
debug('return(ET.tostringlist(data)) didnt work')
return (xmltodict.unparse(data))
if re.match(r'^js(on)?', str(fmt), re.I):
return (json.dumps(data))
if re.match(r'^pretty', str(fmt), re.I):
return (pformat(data))
def create_xml(self, to_file=None):
env = os.environ
# create the file structure
root = ET.Element(env['USERNAME'])
os_items = ET.SubElement(root, 'system')
for key in env.keys():
if 'OS' in key or 'PROCESSOR' in key or 'LC' in key:
system_item = ET.SubElement(os_items, 'system_attribute')
system_item.set('name', key)
system_item.text = env[key]
mydata = ET.tostringlist(root, encoding="unicode", method='xml')
if to_file:
# create a new XML file with the results
with open("computer.xml", "w") as xml_file:
xml_file.writelines(mydata)
return mydata
def write_adaptor_declaration(self, interface_el, notifications, request_responses, out):
global prefix_class_item
def glue_strings(strings):
ret = list()
curstr = ""
for str in strings:
curstr = curstr + str
if str[-1] == ">":
ret.append(curstr)
curstr = ""
return ret
ifacename = interface_el.get("name")
out.write("class " + ifacename + "Adaptor : public QDBusAbstractAdaptor {\n")
out.write(" Q_OBJECT\n")
out.write(' Q_CLASSINFO("D-Bus Interface", "' + self.interface_path + "." + ifacename + '")\n')
out.write(' Q_CLASSINFO("D-Bus Introspection",\n')
introspection_el = self.create_introspection_iface_el(interface_el, "hmi")
introspection = glue_strings(ElementTree.tostringlist(introspection_el))
for str in introspection:
str = str.replace('"', '\\"')
out.write('"' + str + '"' + "\n")
out.write(" )\n")
out.write(" public:\n")
out.write(" explicit " + ifacename + "Adaptor(QObject *parent = 0);\n")
out.write(" void SetApi(Q%sItem*);\n" % prefix_class_item)
out.write(" DBusController *dbusController;\n")
out.write(" public slots:\n")
for (request, response) in request_responses:
signature = self.make_method_signature(request, response, ifacename, False)
out.write(" " + signature + ";\n")
out.write(" signals:\n")
for n in notifications:
signature = self.make_signal_signature(n, ifacename, True)
out.write(" " + signature + ";\n")
out.write(" private slots:\n")
for n in notifications:
signature = self.make_qml_signal_signature(n, ifacename, n.get("name") + "_qml", False)
out.write(" " + signature + ";\n")
out.write(" private:\n")
out.write(" Q%sItem* api_;\n" % prefix_class_item)
out.write("};\n\n")
def getDescription(self, name):
child = self.getTrait(name)
if child == None:
return None
text = ET.tostringlist(child)
# find end of <trait>
i = 0
test = True
while i < len(text) and test:
if '>' in text[i]:
test = False
i += 1
# find start of </trait>
test = True
j = len(text) - 1
while j > 0 and test:
if '</' in text[j]:
test = False
j -= 1
return string.join(text[i+1:j])
def cancel_tasks(self, gateway, task_uris=None):
'''Cancel queued tasks
'''
if not hasattr(gateway, 'tasks'):
return []
if task_uris is None:
task_uris = [task.href for task in gateway.tasks.task]
try:
for task_uri in task_uris:
self.driver.connection.request(task_uri + '/action/cancel',
method='POST')
except Exception as e:
log.error('Error cancelling task %r:', task_uri)
for line in ET.tostringlist(e.args[0]):
log.error(line)
raise
return task_uris
def dumps(self):
''' Dump contents as a string in X3D format '''
xml = []
# Header
xml += ['<?xml version="1.0" encoding="UTF-8"?>\n']
xml += ['<!DOCTYPE %s>\n\n' % X3D_DOCTYPE]
# X3D body
xml += ET.tostringlist(self._root)
# reformat for multi-line printing
tag = None
lined = []
for chunk in xml:
lined.append(chunk)
if chunk.startswith('</'):
tag = 'close'
elif chunk.startswith('<'):
tag = 'open'
elif chunk.endswith('>'):
lined.append('\n')
tag = None
return ''.join(lined)
def save_xml_result(self):
if not self.ns.xmlpath and not self.testsuite_xml:
return
import xml.etree.ElementTree as ET
root = ET.Element("testsuites")
# Manually count the totals for the overall summary
totals = {'tests': 0, 'errors': 0, 'failures': 0}
for suite in self.testsuite_xml:
root.append(suite)
for k in totals:
try:
totals[k] += int(suite.get(k, 0))
except ValueError:
pass
for k, v in totals.items():
root.set(k, str(v))
xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath)
with open(xmlpath, 'wb') as f:
for s in ET.tostringlist(root):
f.write(s)
def toSvg(self, onlySelected=False):
'Return selected or all items in the scene as an SVG document string'
if onlySelected:
import q2str
self._log.trace('selectionGroup: pos={}, transform={}'
, self.scene.selectionGroup.pos()
, q2str.FormatQTransform(self.scene.selectionGroup.transform()) )
svg_attribs = \
{ 'xmlns' : SVG_ns
, 'xmlns:tiles' : TILES_ns
, 'version' : '1.1'
, 'width' : str(self.scene.sceneRect().width())
, 'height' : str(self.scene.sceneRect().height())
}
doc = ET.Element('svg', attrib=svg_attribs)
scene_g = ET.SubElement(doc, 'g', transform='rotate({})'.format(self.graphicsView.GetRoll()))
for it in self.scene.items(order=QtCore.Qt.AscendingOrder):
if it.isSelected() or not onlySelected:
if hasattr(it, 'toSvg'):
e = it.toSvg(scene_g)
scene_g.append(e)
contents = '\n'.join([XML_decl] + ET.tostringlist(doc, encoding='unicode'))
self._log.trace('{}', contents)
return contents
def generate_tag(tag, content, attributes=None):
"""Generate a complete html tag using the ElementTree module. tag and
content are strings, the attributes argument is a dictionary. As
a convenience, if the content is ' /', a self-closing tag is generated."""
content = six.text_type(content)
# In PY2, ElementTree tostringlist only works with bytes, not with
# unicode().
enc = 'unicode'
if six.PY2:
enc = 'UTF-8'
if not tag:
return content
element = ElementTree.Element(tag, attrib=attributes)
# FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried
# adding text by assigning it to element_tag.text. That results in
# non-ascii text being html-entity encoded. Not bad, but not entirely
# matching php-textile either.
element_tag = ElementTree.tostringlist(element, encoding=enc,
method='html')
if six.PY2:
element_tag = [v.decode(enc) for v in element_tag]
element_tag.insert(len(element_tag) - 1, content)
element_text = ''.join(element_tag)
return element_text
def WriteXMLTree(Tree,OutputFile) :
rec=ET.tostringlist(Tree)
fOut=open(OutputFile,'w')
fOut.write('<?xml version="1.0"?>\n')
fOut.write('<!DOCTYPE XSIL SYSTEM "http://www.vallis.org/lisa-xml.dtd">\n')
fOut.write('<?xml-stylesheet type="text/xsl" href="lisa-xml.xsl"?>\n')
Indent=0
xline=""
for ir in xrange(len(rec)) :
xr=rec[ir]
xline = xline + xr
if re.search('</',xr) or ir==1 :
xline = xline + "\n"
if re.search("\n",xline) :
blockType=TypeBlockLine(xline)
if blockType=="close" :
Indent = Indent - 1
xlineI = ReIndent(xline,Indent)
#print Indent,blockType, ">>>>"+xlineI+"<<<<"
fOut.write(xlineI)
xline = ""
if blockType=="open" :
Indent = Indent + 1
fOut.close()
def getstat(statname, number):
return ET.tostringlist(tree.findall("item/" + statname)[number])[2].decode("utf-8")
def getval(tree,value,number):
return ET.tostringlist(tree.findall('item/' + value)[number])[2].decode('utf-8')
lasthai = int(item.tag[1:])
hai[w].remove(int(item.tag[1:]))
if not tp[w]:
machi=calc_shanten(hai[w], naki[w])
if flag_rch:
nodes=ET.SubElement(games,"REACH")
nodes.set("junme",str(junme[w]))
nodes.set("who",str(w))
nodes.set("machi_hai",','.join([str(x) for x in machi[1]]))
flag_rch = False
tp[w] = True
elif machi[0] == 0 and not tp[w]:
tp[w] = True
if naki[w]:
nodes=ET.SubElement(games,"TENPAI")
nodes.set("junme",str(junme[w]))
nodes.set("who",str(w))
nodes.set("machi_hai",','.join([str(x) for x in machi[1]]))
else:
nodes=ET.SubElement(games,"DAMA")
nodes.set("junme",str(junme[w]))
nodes.set("who",str(w))
nodes.set("machi_hai",','.join([str(x) for x in machi[1]]))
continue
cvtlog.writelines(ET.tostringlist(games,encoding="utf-8"))
# print(ET.tostringlist(games,encoding="unicode"))
root.remove(games)
cvtlog.writelines('</mjlog>')
cvtlog.close
import xml.etree.ElementTree as ET
def setelement(theelement, thetext):
ET.SubElement(newitem, theelement).text = thetext
def newelement(whatelement):
setelement(whatelement, str(input(whatelement + ': ')))
while True:
tree = ET.parse('D:\\ENCOUNTER\\ITEMTHING.xml')
root = tree.getroot()
weaponTotal = int(ET.tostringlist(tree.findall("item")[-1])[1].decode("utf-8")[5:][:-1])
newname = str(input("NAME: "))
newdesc = str(input("DESCRIPTION : "))
newitem = ET.SubElement(root,"item")
newitem.set("id", str(weaponTotal+1))
setelement("name", newname)
setelement("desc", newdesc)
ET.ElementTree(root).write("D:\\ENCOUNTER\\ITEMTHING.xml")
def itemtotal(tree):
return int(ET.tostringlist(tree.findall('item')[-1])[1].decode('utf-8')[5:][:-1])
if __name__ == '__main__':
class TestTests(unittest.TestCase):
def test_pass(self):
pass
def test_pass_slow(self):
time.sleep(1.0)
def test_fail(self):
print('stdout', file=sys.stdout)
print('stderr', file=sys.stderr)
self.fail('failure message')
def test_error(self):
print('stdout', file=sys.stdout)
print('stderr', file=sys.stderr)
raise RuntimeError('error message')
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestTests))
stream = io.StringIO()
runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv))
runner = runner_cls(sys.stdout)
result = runner.run(suite)
print('Output:', stream.getvalue())
print('XML: ', end='')
for s in ET.tostringlist(result.get_xml_element()):
print(s.decode(), end='')
print()
# Parse each table
for i, incendio in enumerate(incendios):
logging.info('Parsing fire %s/%s' % (i+1, len(incendios)))
data = {}
# Get the individual data rows
info = incendio.findall("tr")
## Basic data
basic = info[1].findall("td/table/tr")[1].findall("td")
# Extract coordinates from map url
coord_match = re.search(r"x=(?P<x>[0-9.]+)&y=(?P<y>[0-9.]+)", basic[0].find('a').attrib['href'].replace(',', '.'))
data['id'] = clean(ElementTree.tostringlist(basic[1].find('span'))[5])
pt_x = coord_match.group('x')
pt_y = coord_match.group('y')
lisboa_militar = pyproj.Proj("+init=epsg:20790 +nadgrids=pt73_e89.gsb")
wgs84 = pyproj.Proj("+init=epsg:4326")
wgs_x, wgs_y = pyproj.transform(lisboa_militar, wgs84, pt_x, pt_y)
data['coordinates'] = {
'x': wgs_x,
'y': wgs_y
}
data['location'] = clean(ElementTree.tostringlist(basic[1].find('span'))[2])
data['concelho'] = basic[4].find('span').text
def run(self):
# Read CSV data and replace the variables in the Scribus File with the cooresponding data. Finaly export to the specified format.
# may throw exceptions if errors are met, use traceback to get all error details
#defaults for missing info
if(self.__dataObject.getSingleOutput() and (self.__dataObject.getOutputFileName() is CONST.EMPTY)):
self.__dataObject.setOutputFileName(os.path.split(os.path.splitext(self.__dataObject.getScribusSourceFile())[0])[1] +'__single')
#parsing
logging.debug("parsing data source file %s"%(self.__dataObject.getDataSourceFile()))
csvData = self.getCsvData(self.__dataObject.getDataSourceFile())
if(len(csvData) < 1):
logging.error("Data file %s is empty. At least a header line and a line of data is needed. Halting."%(self.__dataObject.getDataSourceFile()))
return -1
if(len(csvData) < 2):
logging.error("Data file %s has only one line. At least a header line and a line of data is needed. Halting."%(self.__dataObject.getDataSourceFile()))
return -1
#range
firstElement = 1
if(self.__dataObject.getFirstRow() != CONST.EMPTY):
try:
newFirstElementValue = int(self.__dataObject.getFirstRow())
firstElement = max(newFirstElementValue, 1) # Guard against 0 or negative numbers
except:
logging.warning("Could not parse value of 'first row' as an integer, using default value instead")
lastElement = len(csvData)
if(self.__dataObject.getLastRow() != CONST.EMPTY):
try:
newLastElementValue = int(self.__dataObject.getLastRow())
lastElement = min(newLastElementValue + 1, lastElement) # Guard against numbers higher than the length of csvData
except:
logging.warning("Could not parse value of 'last row' as an integer, using default value instead")
if ( (firstElement != 1) or (lastElement != len(csvData)) ):
csvData = csvData[0:1] + csvData[firstElement : lastElement]
#generation
dataC = len(csvData)-1
fillCount = len(str(dataC))
template = [] # XML-Content/Text-Content of the Source Scribus File (List of Lines)
outputFileNames = []
index = 0
# Generate the Scribus Files
for row in csvData:
if(index == 0): # first line is the Header-Row of the CSV-File
headerRowForFileName = row
headerRowForReplacingVariables = self.handleAmpersand(row) # Header-Row contains the variable names
# overwrite attributes from their /*/ItemAttribute[Type=SGAttribute] sibling, when applicable.
logging.debug("parsing scribus source file %s"%(self.__dataObject.getScribusSourceFile()))
tree = ET.parse(self.__dataObject.getScribusSourceFile())
root = tree.getroot()
templateElt = self.overwriteAttributesFromSGAttributes(root)
else:
outContent = self.replaceVariablesWithCsvData(headerRowForReplacingVariables, self.handleAmpersand(row), ET.tostringlist(templateElt))
if (self.__dataObject.getSingleOutput()):
if (index == 1):
logging.debug("generating reference content from row #1")
outputElt = ET.fromstring(outContent)
docElt = outputElt.find('DOCUMENT')
pagescount = int(docElt.get('ANZPAGES'))
pageheight = float(docElt.get('PAGEHEIGHT'))
vgap = float(docElt.get('GapVertical'))
groupscount = int(docElt.get('GROUPC'))
objscount = len(outputElt.findall('.//PAGEOBJECT'))
logging.debug("current template has #%s pageobjects"%(objscount))
version = outputElt.get('Version')
# if version.startswith('1.4'):
# docElt.set('GROUPC', str(groupscount*dataC))
docElt.set('ANZPAGES', str(pagescount*dataC))
docElt.set('DOCCONTRIB',docElt.get('DOCCONTRIB')+CONST.CONTRIB_TEXT)
else:
logging.debug("merging content from row #%s"%(index))
tmpElt = ET.fromstring(outContent).find('DOCUMENT')
shiftedElts = self.shiftPagesAndObjects(tmpElt, pagescount, pageheight, vgap, index-1, groupscount, objscount, version)
docElt.extend(shiftedElts)
else:
outputFileName = self.createOutputFileName(index, self.__dataObject.getOutputFileName(), headerRowForFileName, row, fillCount)
scribusOutputFilePath = self.createOutputFilePath(self.__dataObject.getOutputDirectory(), outputFileName, CONST.FILE_EXTENSION_SCRIBUS)
self.exportSLA(scribusOutputFilePath, outContent)
outputFileNames.append(outputFileName)
logging.info("scribus file created: %s"%(scribusOutputFilePath))
index = index + 1
# write single sla
if (self.__dataObject.getSingleOutput()):
scribusOutputFilePath = self.createOutputFilePath(self.__dataObject.getOutputDirectory(), self.__dataObject.getOutputFileName(), CONST.FILE_EXTENSION_SCRIBUS)
outTree = ET.ElementTree(outputElt)
outTree.write(scribusOutputFilePath, encoding="UTF-8")
outputFileNames.append(self.__dataObject.getOutputFileName())
logging.info("scribus file created: %s"%(scribusOutputFilePath))
# Export the generated Scribus Files as PDF
if(CONST.FORMAT_PDF == self.__dataObject.getOutputFormat()):
for outputFileName in outputFileNames:
pdfOutputFilePath = self.createOutputFilePath(self.__dataObject.getOutputDirectory(), outputFileName, CONST.FILE_EXTENSION_PDF)
scribusOutputFilePath = self.createOutputFilePath(self.__dataObject.getOutputDirectory(), outputFileName, CONST.FILE_EXTENSION_SCRIBUS)
self.exportPDF(scribusOutputFilePath, pdfOutputFilePath)
logging.info("pdf file created: %s"%(pdfOutputFilePath))
# Cleanup the generated Scribus Files
if(not (CONST.FORMAT_SLA == self.__dataObject.getOutputFormat()) and CONST.FALSE == self.__dataObject.getKeepGeneratedScribusFiles()):
for outputFileName in outputFileNames:
scribusOutputFilePath = self.createOutputFilePath(self.__dataObject.getOutputDirectory(), outputFileName, CONST.FILE_EXTENSION_SCRIBUS)
self.deleteFile(scribusOutputFilePath)
return 1;
