def save_element(elem, path):
if elem is None:
return
if not path or path is None:
return
parent = os.path.dirname(path)
if not os.path.isdir(parent):
os.makedirs(parent, mode=0o755, exist_ok=True)
os.chmod(parent, 0o0755)
try:
# Get the elements default namespace
namespace = getNamespace(elem)
# Pretty up the element
indent(elem)
# Register the discovered namespace as the default
ET.register_namespace("", namespace)
# Create a new ElementTree with this element as the root
elem_tree = ET.ElementTree(element=elem)
# Write the full tree to a file
elem_tree.write(path, xml_declaration=False, encoding="UTF-8", method="xml")
os.chmod(path, 0o0664)
return True
except:
print(" *** Error writing new element to path %s" % path)
print(" Exception Type: ", sys.exc_info()[0])
print(" Exception Value: ", sys.exc_info()[1])
return False
def limit_rdb(services_rdb, full_factory_map, full_constructor_map):
ET.register_namespace('','http://openoffice.org/2010/uno-components')
tree = ET.parse(services_rdb[0])
root = tree.getroot()
for component in root.findall('{http://openoffice.org/2010/uno-components}component'):
# direct
uri = component.get('uri')
component_name = None
if uri != None:
component_name = re.sub('^vnd.sun.star.expand:\$LO_LIB_DIR/([^.]*).so$', '\\1.a', uri)
if component_name in full_factory_map:
continue
# via a constructor - limit only to those we have
has_constructor = False
for implementation in component.findall('{http://openoffice.org/2010/uno-components}implementation'):
constructor = implementation.get('constructor')
if constructor in full_constructor_map:
has_constructor = True
else:
component.remove(implementation)
if not has_constructor:
root.remove(component)
tree.write(services_rdb[0] + '.out', xml_declaration = True, method = 'xml')
def get_bib_string(bib_xml):
"""
this function extracts the bibliographic record from
the XML string returned by ALMA's API. ALMA's
MARCXML record is enclosed by markup
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record></record>
</collection>
"""
bib_string=""
outcome=1
in_string=bib_xml.replace("\n","")
try:
tree=elementTree.fromstring(in_string)
except:
sys.stderr.write("parse failed."+"\n")
return bib_string,outcome
try:
bib_element=tree.find("record")
except:
sys.stderr.write("xml parsing failed. couldn't find record element"+"\n")
return bib_string,outcome
try:
bib_string=elementTree.tostring(bib_element,encoding="utf-8",method="xml")
#sys.stderr.write(str(bib_string)+"\n"))
bib_string=bib_string.replace("<?xml version='1.0' encoding='utf-8'?>","")
#bib_string=elementTree.tostring(bib_element,method="xml")
except:
sys.stderr.write("xml parsing failed. couldn't find record element"+"\n")
return "",outcome
bib_string=bib_string.replace(" schemaLocation=\"http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd\"","")
bib_string=bib_string.replace("</record>","")
bib_string=bib_string.replace("<leader> ","<leader>12345")
return bib_string,0
def test_postXMLEndpoint(self):
testURL = "https://api.eveonline.com/server/ServerStatus.xml.aspx"
res_cache = api.postXMLEndpoint(testURL)
self.assertIsInstance(res_cache, ET.Element)
# Test cached response
self.assertEqual(ET.tostring(api.postXMLEndpoint(testURL)), ET.tostring(res_cache))
def test_affiliate_with_supported_seats(self):
""" Verify that endpoint returns course runs for verified and professional seats only. """
response = self.client.get(self.affiliate_url)
self.assertEqual(response.status_code, 200)
root = ET.fromstring(response.content)
self.assertEqual(1, len(root.findall('product')))
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, self.seat_verified.type))[0],
self.seat_verified
)
# Add professional seat.
seat_professional = SeatFactory(course_run=self.course_run, type=Seat.PROFESSIONAL)
response = self.client.get(self.affiliate_url)
root = ET.fromstring(response.content)
self.assertEqual(2, len(root.findall('product')))
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, self.seat_verified.type))[0],
self.seat_verified
)
self.assert_product_xml(
root.findall('product/[pid="{}-{}"]'.format(self.course_run.key, seat_professional.type))[0],
seat_professional
)
def isIgnored(myMedia):
ignored = False
msg = u'File: {0}'.format(myMedia.oStatus.fileName)
Debug(msg)
pchtrakt.logger.info(msg)
ignored = isKeywordIgnored(myMedia.oStatus.fileName)
if not ignored and ignored_repertory[0] != '':
for el in myMedia.oStatus.fullPath.split('/'):
if el <> '' and el in ignored_repertory:
msg = 'This video is in a ignored repertory: {0}'.format(el)
ignored = True
break
if not ignored and YamjIgnoredCategory[0] != '':
if isinstance(myMedia.parsedInfo, mp.MediaParserResultTVShow):
files = listdir(YamjPath)
for file in files:
if file.endswith('xml'):
file = unicode(file, errors='replace')
if file.find(myMedia.parsedInfo.name) >= 0:
oXml = ElementTree.parse(YamjPath + file)
ignored = isGenreIgnored(oXml.findall('.//genre'))
if ignored:
break
else:
file = unicode(myMedia.oStatus.fileName.rsplit('.',1)[0] + '.xml', errors='replace')
oXml = ElementTree.parse(YamjPath + file)
genres = oXml.findall('.//genre')
ignored = isGenreIgnored(genres)
return ignored
def set_owner(conn, uuid, owner):
"""Set ON_OWNER by uuid.
@param uuid: uuid of the VM
@param owner: string representing owner
"""
vmid = get_id_by_uuid(conn, uuid)
if not vmid:
return
vm = conn.lookupByID(vmid)
domain = ET.fromstring(vm.XMLDesc(0))
metadata = domain.find('./metadata') or ET.SubElement(domain, 'metadata')
owner_e = ET.SubElement(metadata, ET.QName('urn:opennode-tui', 'owner'))
owner_e.text = owner
## TODO: cleanup
open('/etc/libvirt/qemu/%s.xml' % (vm.name()), 'w').write(ET.tostring(domain))
data = open('/etc/libvirt/qemu/%s.xml' % (vm.name()), 'r').read()
domain_n = ET.fromstring(data)
owner_e = domain_n.find('./metadata/{urn:opennode-tui}owner')
assert owner_e is not None
assert owner_e.text == owner
return owner_e.text
def mount_iso(self, dev, image):
def attach_iso(dev, disk, vol):
if disk.get('device') == 'cdrom':
for elm in disk:
if elm.tag == 'target':
if elm.get('dev') == dev:
src_media = ElementTree.Element('source')
src_media.set('file', vol.path())
disk.insert(2, src_media)
return True
storages = self.get_storages()
for storage in storages:
stg = self.get_storage(storage)
if stg.info()[0] != 0:
for img in stg.listVolumes():
if image == img:
vol = stg.storageVolLookupByName(image)
tree = ElementTree.fromstring(self._XMLDesc(0))
for disk in tree.findall('devices/disk'):
if attach_iso(dev, disk, vol):
break
if self.get_status() == 1:
xml = ElementTree.tostring(disk)
self.instance.attachDevice(xml)
xmldom = self._XMLDesc(VIR_DOMAIN_XML_SECURE)
if self.get_status() == 5:
xmldom = ElementTree.tostring(tree)
self._defineXML(xmldom)
def callback(results):
reactor.stop()
for success, result in results:
if success:
print ElementTree.dump(result)
else:
print result.printTraceback()
def data_import(self, odm):
"""
Return result of request to import data to an instance.
Required params:
:param odm: odm xml data for import, must be utf-8 encoded and unescaped.
:type odm: str.
:returns: contents of the method response in an OrderedDict, or None.
"""
envelope_copy = et.fromstring(self.envelope)
body = [i for i in envelope_copy.iterfind(
'.//se:Body', {'se': self.ns_se})][0]
method_name = 'importRequest'
import_request = et.Element(et.QName(self.ns_data, method_name))
body.append(import_request)
odm_node = et.Element('odm')
odm_node.append(et.fromstring(odm))
import_request.append(odm_node)
envelope = et.tostring(envelope_copy)
response = self.request(
self.ocws_url, envelope, method_name, self.ns_data)
return response
def fix_graphml_format_better(cls, name, gridfs):
cls._logger.info("Reformatting file to fix GDO graphml formatting using element tree")
replacements = {}
with gridfs.get_last_version(name) as f:
tree = ElementTree.parse(f)
f_id = f._id
for key_entry in tree.findall("{http://graphml.graphdrawing.org/xmlns}key"):
key_name = key_entry.attrib['attr.name']
id = key_entry.attrib['id']
key_entry.attrib['id'] = key_name
replacements[id] = key_name
# cls._logger.info("Name: %s, Key %s", key_name, key_entry)
# cls._logger.info("Dict %s", replacements)
root=tree.getroot()
for data_entry in root.iter("{http://graphml.graphdrawing.org/xmlns}data"):
found_key = data_entry.attrib['key']
data_entry.set('key', replacements[found_key])
ElementTree.register_namespace('', "http://graphml.graphdrawing.org/xmlns")
with gridfs.new_file(filename=name, content_type="text/xml") as des:
tree.write(des, encoding='utf-8', method='xml')
cls._logger.info("Deleting old version of file %s", f_id)
gridfs.delete(f_id)
def open_soap_envelope(text):
"""
:param text: SOAP message
:return: dictionary with two keys "body"/"header"
"""
try:
envelope = ElementTree.fromstring(text)
except Exception as exc:
raise XmlParseError("%s" % exc)
assert envelope.tag == '{%s}Envelope' % soapenv.NAMESPACE
assert len(envelope) >= 1
content = {"header": [], "body": None}
for part in envelope:
if part.tag == '{%s}Body' % soapenv.NAMESPACE:
assert len(part) == 1
content["body"] = ElementTree.tostring(part[0], encoding="UTF-8")
elif part.tag == "{%s}Header" % soapenv.NAMESPACE:
for item in part:
_str = ElementTree.tostring(item, encoding="UTF-8")
content["header"].append(_str)
return content
def add_to_manifest(repositories, fallback_branch = None):
try:
lm = ElementTree.parse(".repo/local_manifests/du_manifest.xml")
lm = lm.getroot()
except:
lm = ElementTree.Element("manifest")
for repository in repositories:
repo_name = repository['repository']
repo_target = repository['target_path']
if exists_in_tree(lm, repo_name):
print('DirtyUnicorns/%s already exists' % (repo_name))
continue
print('Adding dependency: DirtyUnicorns/%s -> %s' % (repo_name, repo_target))
project = ElementTree.Element("project", attrib = { "path": repo_target,
"remote": "du", "name": "%s" % repo_name })
if 'branch' in repository:
project.set('revision',repository['branch'])
elif fallback_branch:
print("Using fallback branch %s for %s" % (fallback_branch, repo_name))
project.set('revision', fallback_branch)
else:
print("Using default branch for %s" % repo_name)
lm.append(project)
indent(lm, 0)
raw_xml = ElementTree.tostring(lm).decode()
raw_xml = '<?xml version="1.0" encoding="UTF-8"?>\n' + raw_xml
f = open('.repo/local_manifests/du_manifest.xml', 'w')
f.write(raw_xml)
f.close()
def test_remove_doubletheme(self):
context = ET.fromstring("""
<contexte xmlns="http://olst.ling.umontreal.ca/dicoenviro/">
<contexte-texte>I compare apples and oranges</contexte-texte>
<participant type="Act" role="Agent">
<fonction-syntaxique nom="Subject">
<groupe-syntaxique nom="NP">I</groupe-syntaxique>
</fonction-syntaxique>
</participant>
<lexie-att>compare</lexie-att>
<participant type="Act" role="Theme">
<fonction-syntaxique nom="Object">
<groupe-syntaxique nom="NP">apples</groupe-syntaxique>
</fonction-syntaxique>
</participant>
and
<participant type="Act" role="Theme">
<fonction-syntaxique nom="Object">
<groupe-syntaxique nom="NP">oranges</groupe-syntaxique>
</fonction-syntaxique>
</participant>
</contexte>
""")
sentence_text, frame = xmlcontext_to_frame(
"http://olst.ling.umontreal.ca/dicoenviro/", 'compare', context)
vn_frame = ET.fromstring(
"""<SYNTAX><NP value="Agent" /><VERB /><NP value="Theme" /></SYNTAX>""")
self.assertEqual(syntax_to_str(frame), syntax_to_str(vn_frame))
def parse_soap_enveloped_saml_thingy(text, expected_tags):
"""Parses a SOAP enveloped SAML thing and returns the thing as
a string.
:param text: The SOAP object as XML string
:param expected_tags: What the tag of the SAML thingy is expected to be.
:return: SAML thingy as a string
"""
envelope = ElementTree.fromstring(text)
# Make sure it's a SOAP message
assert envelope.tag == '{%s}Envelope' % soapenv.NAMESPACE
assert len(envelope) >= 1
body = None
for part in envelope:
if part.tag == '{%s}Body' % soapenv.NAMESPACE:
assert len(part) == 1
body = part
break
if body is None:
return ""
saml_part = body[0]
if saml_part.tag in expected_tags:
return ElementTree.tostring(saml_part, encoding="UTF-8")
else:
raise WrongMessageType("Was '%s' expected one of %s" % (saml_part.tag,
expected_tags))
def main():
count = 1
start_title = parse_args()
print "%i. %s" % (count, start_title)
count += 1
next_page = start_title
link_list = []
next_page = next_page.replace(" ", "+").encode('utf-8')
xml = urllib2.urlopen(
'https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=links&indexpageids=1&titles=' + next_page + '&plnamespace=&pllimit=5000')
root = ET.fromstring(xml.read())
for child in root.iter('pl'):
if ':' not in child.get('title'):
link_list.append(child.get('title'))
next_page = random.choice(link_list)
while next_page:
last_list = link_list
link_list = []
while not link_list:
next_page = random.choice(last_list)
print "%i. %s" % (count, next_page)
count += 1
next_page = next_page.replace(" ", "+").encode('utf-8')
xml = urllib2.urlopen(
'https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=links&indexpageids=1&titles=' + next_page + '&plnamespace=&pllimit=5000')
root = ET.fromstring(xml.read())
for child in root.iter('pl'):
if ':' not in child.get('title'):
link_list.append(child.get('title'))
def ensure_elementtree_imported(verbosity, logfile):
global ET, ET_has_iterparse
if ET is not None:
return
if "IronPython" in sys.version:
import xml.etree.ElementTree as ET
#### 2.7.2.1: fails later with
#### NotImplementedError: iterparse is not supported on IronPython. (CP #31923)
else:
try: import xml.etree.cElementTree as ET
except ImportError:
try: import cElementTree as ET
except ImportError:
try: import lxml.etree as ET
except ImportError:
try: import xml.etree.ElementTree as ET
except ImportError:
try: import elementtree.ElementTree as ET
except ImportError:
raise Exception("Failed to import an ElementTree implementation")
if hasattr(ET, 'iterparse'):
_dummy_stream = BYTES_IO(b'')
try:
ET.iterparse(_dummy_stream)
ET_has_iterparse = True
except NotImplementedError:
pass
if verbosity:
etree_version = repr([
(item, getattr(ET, item))
for item in ET.__dict__.keys()
if item.lower().replace('_', '') == 'version'
])
print(ET.__file__, ET.__name__, etree_version, ET_has_iterparse, file=logfile)
def xml(self):
"""
xml representation of the metadata.
:return: xml representation of the metadata
:rtype: ElementTree.Element
"""
root = super(ImpactLayerMetadata, self).xml
provenance_path = self._special_properties['provenance']
provenance_element = root.find(provenance_path, XML_NS)
# find the provenance parent tag
if provenance_element is not None:
# there is already a provenance tag so we remove it
provenance_parent = provenance_element.getparent()
provenance_parent.remove(provenance_element)
else:
# find the parent using the provenance path minus one level
provenance_parent = '/'.join(provenance_path.split('/')[:-1])
provenance_parent = root.find(provenance_parent, XML_NS)
# generate the provenance xml element
provenance_element = ElementTree.fromstring(self.provenance.xml)
provenance_parent.append(provenance_element)
return prettify_xml(ElementTree.tostring(root))
def setUp(self):
JSBSimTestCase.setUp(self)
self.fdm = CreateFDM(self.sandbox)
self.script_path = self.sandbox.path_to_jsbsim_file('scripts',
'c1722.xml')
# Read the time step 'dt' from the script file
self.tree = et.parse(self.script_path)
root = self.tree.getroot()
use_tag = root.find('use')
aircraft_name = use_tag.attrib['aircraft']
self.run_tag = root.find('run')
self.dt = float(self.run_tag.attrib['dt'])
# Read the date at which the trim will be run
for event in root.findall('run/event'):
if event.attrib['name'] == 'Trim':
cond_tag = event.find('condition')
self.trim_date = float(cond_tag.text.split()[-1])
break
# Read the output rate and the output file from the aircraft file
aircraft_path = self.sandbox.path_to_jsbsim_file('aircraft', aircraft_name,
append_xml(aircraft_name))
tree = et.parse(aircraft_path)
output_tag = tree.getroot().find('output')
self.output_file = output_tag.attrib['name']
self.rateHz = float(output_tag.attrib['rate'])
self.rate = int(1.0 / (self.rateHz * self.dt))
def to_string(self, endpoints):
"""
Converts the given endpoint description beans into a string
:param endpoints: A list of EndpointDescription beans
:return: A string containing an XML document
"""
# Make the ElementTree
root = self._make_xml(endpoints)
tree = ElementTree.ElementTree(root)
# Force the default name space
ElementTree.register_namespace("", EDEF_NAMESPACE)
# Make the XML
for encoding in ('unicode', 'UTF-8'):
# Prepare a StringIO output
output = StringIO()
try:
# Try to write with a correct encoding
tree.write(output, encoding=encoding, xml_declaration=True,
method="xml")
break
except LookupError:
# 'unicode' is needed in Python 3, but unknown in Python 2...
continue
else:
raise LookupError("Couldn't find a valid encoding")
return output.getvalue()
def hilight(self, buttons):
""" Hilights specified button, if same ID is found in svg """
cache_id = "|".join([ "%s:%s" % (x, buttons[x]) for x in buttons ])
if not cache_id in self.cache:
# Ok, this is close to madness, but probably better than drawing
# 200 images by hand;
if len(buttons) == 0:
# Quick way out - changes are not needed
svg = Rsvg.Handle.new_from_data(self.current_svg.encode("utf-8"))
else:
# 1st, parse source as XML
tree = ET.fromstring(self.current_svg.encode("utf-8"))
# 2nd, change colors of some elements
for button in buttons:
el = SVGEditor.find_by_id(tree, button)
if el is not None:
SVGEditor.recolor(el, buttons[button])
# 3rd, turn it back into XML string......
xml = ET.tostring(tree)
# ... and now, parse that as XML again......
svg = Rsvg.Handle.new_from_data(xml.encode("utf-8"))
while len(self.cache) >= self.CACHE_SIZE:
self.cache.popitem(False)
if self.size_override:
w, h = self.size_override
self.cache[cache_id] = svg.get_pixbuf().scale_simple(
w, h, GdkPixbuf.InterpType.BILINEAR)
else:
self.cache[cache_id] = svg.get_pixbuf()
self.image.set_from_pixbuf(self.cache[cache_id])
def _get_game_xml(self, steam_id, mode):
"""Get the XML list of games"""
fname = "games_{}.xml".format(steam_id)
if mode == "read":
log.info("Reading from {}".format(fname))
with open(fname, 'rb') as f:
content = f.read()
else:
url = self.userUrl.format(steam_id=steam_id)
log.debug("Requesting {}".format(url))
f = urlopen(url)
content = f.read()
if mode == "write":
log.info("Writing to {}".format(fname))
with open(fname, 'wb') as f:
f.write(content)
try:
root = ET.fromstring(content)
except UnicodeEncodeError:
root = ET.fromstring(content.encode('utf-8'))
return root
def multicopy():
try:
tree = ET.parse('filecopylocations.xml')
xroot = tree.getroot()
print "\nWould you like to edit the following copy desitinations?\n"
ET.dump(xroot)
edit = raw_input("\ny=yes : n=no\n")
if edit == 'y':
initMulticopy()
else:
pass
except:
initMulticopy()
tree = ET.parse('filecopylocations.xml')
xroot = tree.getroot()
print "\nPlease select the file you wish to have copied."
try:
fcpyfrom = getFileLocation()
fcpyname = getFileNameFromFilePath(fcpyfrom)
fcpyfrom = open(fcpyfrom, 'r').read()
for loc in xroot.findall('Location'):
f = open(loc.text + '/' + fcpyname, 'w')
f.write(fcpyfrom)
f.close()
print "\nFile was successfully copied!"
except:
print "\nCould not copy file!"
pass
def run(self):
"""Runner"""
service_path = rel_path(os.path.join("build",
"Add to BibDesk.workflow", "Contents", "document.wflow"))
app_path = rel_path(os.path.join("build", "ADS to BibDesk.app",
"Contents", "document.wflow"))
py_path = rel_path("adsbibdesk.py")
for workflow in (service_path, app_path):
with open(workflow, 'r') as fh:
xml = ElementTree.fromstring(fh.read())
for arr in xml.find('dict').find('array').getchildren():
# fetch Python code inside the xml
py = [c for c in arr.find('dict').getchildren()
if c.tag == 'dict' and
any([i.text and '/usr/bin/env' in i.text
for i in c.getchildren()])]
# rewrite with current file
if py:
logger.info("Inserting {0} into {1}".format(py_path,
workflow))
py[0].find('string').text = open(py_path).read()
logger.info("Saving {0}".format(workflow))
with open(workflow, 'wb') as fh:
fh.write(ElementTree.tostring(xml))
logger.info("Completed ADS to BibDesk build step")
def intiMulticopy(self):
fio = fIO()
fio.filePrompt("folder", "Please select folder locations to copy to.\n"
+ "These folder locations will be saved.\n"
+ "When you are done loading in locations, simply\n"
+ "press the cancel button in the file explorer.\n")
try:
tree = ET.parse('filecopylocations.xml')
xroot = tree.getroot()
for locs in xroot.findall('Location'):
xroot.remove(locs)
except:
froot = ET.Element('Directories')
tree = ET.ElementTree(froot)
tree.write('filecopylocations.xml')
tree = ET.parse('filecopylocations.xml')
xroot = tree.getroot()
locnum = 1
fio.folderLocation()
floc = fio.getFolderLocation()
while(floc != ''):
try:
loc = ET.SubElement(xroot, 'Location'.format(locnum))
loc.set('index', '{0}'.format(locnum))
locnum = locnum + 1
loc.text = floc
floc = fio.getFolderLocation()
except:
floc = ''
tree.write('filecopylocations.xml')
ET.dump(xroot)
def globalVars():
global _home_dir, _desktop_folder, _script_dir, xml_cache_root, xml_options_root, _cache_dir, _options_dir, qualityPresets
_home_dir = QtCore.QDir.homePath()
_desktop_folder = _home_dir + "/Desktop/"
_script_dir = QtCore.QDir.currentPath()
_cache_dir = _script_dir + "/cache.xml"
_options_dir = _script_dir + "/options.xml"
external_dir = _home_dir + "/.audiobookEncoder"
if not os.path.exists(external_dir):
os.makedirs(external_dir)
if not os.path.exists(external_dir + "/cache.xml"):
shutil.copy2(_cache_dir, external_dir)
if not os.path.exists(external_dir + "/options.xml"):
shutil.copy2(_options_dir, external_dir)
_cache_dir = external_dir + "/cache.xml"
_options_dir = external_dir + "/options.xml"
# xml table auslesen
xml_cache_root = ET.parse(_cache_dir).getroot()
xml_options_root = ET.parse(_options_dir).getroot()
qualityPresets = ["96 Kbps, Stereo, 48 kHz", "128 Kbps, Stereo, 48 kHz", "256 Kbps, Stereo, 48 kHz", "320 Kbps, Stereo, 48 kHz"]
def _create_html_file(self, sourcefile, htmlfile, errors):
name = self.generator.get_name()
root = ElementTree.fromstring(CPPCHECK_HTML_FILE)
title = root.find("head/title")
title.text = "cppcheck - report - %s" % name
body = root.find("body")
for div in body.findall("div"):
if div.get("id") == "page":
page = div
break
for div in page.findall("div"):
if div.get("id") == "header":
h1 = div.find("h1")
h1.text = "cppcheck report - %s" % name
if div.get("id") == "content":
content = div
srcnode = self.generator.bld.root.find_node(sourcefile)
hl_lines = [e["line"] for e in errors if e.has_key("line")]
formatter = CppcheckHtmlFormatter(linenos=True, style="colorful", hl_lines=hl_lines, lineanchors="line")
formatter.errors = [e for e in errors if e.has_key("line")]
css_style_defs = formatter.get_style_defs(".highlight")
lexer = pygments.lexers.guess_lexer_for_filename(sourcefile, "")
s = pygments.highlight(srcnode.read(), lexer, formatter)
table = ElementTree.fromstring(s)
content.append(table)
s = ElementTree.tostring(root, method="html")
s = CCPCHECK_HTML_TYPE + s
node = self.generator.path.get_bld().find_or_declare(htmlfile)
node.write(s)
return css_style_defs
def test_get_xml(self):
self.resModelProgram.metadata.create_element('Description', abstract="test abstract")
self.resModelProgram.metadata.create_element('Subject', value="test subject")
release_date = '2016-10-24T21:05:00.315907+00:00'
self.resModelProgram.metadata.create_element('MpMetadata',
modelVersion='5.1.011',
modelProgramLanguage='Fortran',
modelOperatingSystem='Windows',
modelReleaseDate=release_date,
modelWebsite='http://www.hydroshare.org',
modelCodeRepository='http://www.github.com',
modelReleaseNotes='releaseNote.pdf',
modelDocumentation='manual.pdf',
modelSoftware='utilities.exe',
modelEngine='sourceCode.zip')
# test if xml from get_xml() is well formed
ET.fromstring(self.resModelProgram.metadata.get_xml())
xml_doc = self.resModelProgram.metadata.get_xml()
# check to see if the specific metadata are in the xml doc
self.assertTrue('5.1.011' in xml_doc)
self.assertTrue('Fortran' in xml_doc)
self.assertTrue('Windows' in xml_doc)
self.assertTrue(release_date in xml_doc)
self.assertTrue('http://www.hydroshare.org' in xml_doc)
self.assertTrue('http://www.github.com' in xml_doc)
self.assertTrue('releaseNote.pdf' in xml_doc)
self.assertTrue('manual.pdf' in xml_doc)
self.assertTrue('utilities.exe' in xml_doc)
self.assertTrue('sourceCode.zip' in xml_doc)
def get_product_pages(static,url,logger):
logger.debug("In get_product_pages: " + url)
# Get the svr first (it's global)
lines = themortgagemeter_utils.get_page(False,'',url,logger,True).split('\n')
# Now get the mortgage data
if static:
tree = ET.parse('static_html/tesco/Products.xml')
root = tree.getroot()
else:
root = ET.fromstring(themortgagemeter_utils.get_page(False,'',url,logger,True))
term = str(25 * 12)
for purchase in ('HousePurchase','Remortgage'):
if purchase == 'HousePurchase':
eligibilities = ['NFTB','NMH']
elif purchase == 'Remortgage':
eligibilities = ['NRM']
for rate_type in ('FixedRate','TrackerRate'):
if rate_type == 'FixedRate':
mortgage_type = 'F'
elif rate_type == 'TrackerRate':
mortgage_type = 'T'
rate_set = root.find(purchase).find(rate_type)
for rate in rate_set.findall('LTV'):
ltv_percent = rate.get('max')
for mortgage in rate.findall('Mortgage'):
#ET.dump(mortgage)
#print "--------------------"
rate_percent = mortgage.find('initialRate').text
apr_percent = mortgage.find('APR').text
svr_percent = mortgage.find('variableRate').text
name = mortgage.find('name').text.split('\n')[0]
initial_period = themortgagemeter_utils.get_months(name,logger)
booking_fee = str(int(mortgage.find('bookingFee').text) + int(mortgage.find('productFee').text))
for eligibility in eligibilities:
mc_util.handle_mortgage_insert(institution_code,mortgage_type,rate_percent,svr_percent,apr_percent,ltv_percent,initial_period,booking_fee,term,'http://www.tescobank.com/personal/finance/mortgages',eligibility,logger)
def _clone_menu(self, xml, name, owner=None):
"""
Hopefully temporary hack for converting UIManager XML describing the
main menubar into a rebindable popup menu. UIManager by itself doesn't
let you do this, by design, but we need a bigger menu than the little
things it allows you to build.
"""
ui_elt = ET.fromstring(xml)
rootmenu_elt = ui_elt.find("menubar")
rootmenu_elt.attrib["name"] = name
xml = ET.tostring(ui_elt)
self.app.ui_manager.add_ui_from_string(xml)
tmp_menubar = self.app.ui_manager.get_widget('/' + name)
popupmenu = gtk.Menu()
for item in tmp_menubar.get_children():
tmp_menubar.remove(item)
popupmenu.append(item)
if owner is not None:
popupmenu.attach_to_widget(owner, None)
popupmenu.set_title("MyPaint")
popupmenu.connect("selection-done", self.popupmenu_done_cb)
popupmenu.connect("deactivate", self.popupmenu_done_cb)
popupmenu.connect("cancel", self.popupmenu_done_cb)
self.popupmenu_last_active = None
return popupmenu
if len(sys.argv) <= 1:
raise ValueError('The first argument must be the product.')
product = sys.argv[1]
try:
device = product[product.index('_') + 1:]
except ValueError:
device = product
dependencies_json_path %= device
if not os.path.isfile(dependencies_json_path):
raise ValueError('No dependencies file could be found for the device (%s).' % device)
dependencies = json.loads(open(dependencies_json_path, 'r').read())
try:
upstream_manifest = ET.parse(upstream_manifest_path).getroot()
except (IOError, ET.ParseError):
upstream_manifest = ET.Element('manifest')
recurse_include(upstream_manifest)
try:
roomservice_manifest = ET.parse(roomservice_manifest_path).getroot()
except (IOError, ET.ParseError):
roomservice_manifest = ET.Element('manifest')
syncable_projects = []
# Clean up all the <remove-project> elements.
for removable_project in roomservice_manifest.findall('remove-project'):
name = removable_project.get('name')
def avatar3d(request):
"""Serve an 3d avatar."""
theme = request.GET.get('theme', '3d')
#get request
accept_ids = request.GET.getlist('ids')
if not accept_ids:
accept_ids = request.GET.getlist('ids[]')
skinTone = request.GET.get('skinTone', '')
hairTone = request.GET.get('hairTone', '')
viewBox = request.GET.get('viewBox', '')
height = request.GET.get('height', '')
width = request.GET.get('width', '')
scale = request.GET.get('scale', '')
mode = request.GET.get('mode', '')
height = height if height else scale
width = width if width else scale
force_show_whole_body = request.GET.get('force_show_whole_body', True)
if mode == 'preview' and len(accept_ids) == 1:
height = 30
width = 30
_type = accept_ids[0].split('_')[0]
viewBox = get_avatar_attrs(theme, 'preview_viewbox').get(
_type, '0 0 600 600')
force_show_whole_body = False
else:
accept_ids.append('frame')
# setup
tags = ['{http://www.w3.org/2000/svg}style']
ET.register_namespace('', "http://www.w3.org/2000/svg")
prepend = f'''<?xml version="1.0" encoding="utf-8"?>
<svg width="{width}%" height="{height}%" viewBox="{viewBox}" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
'''
postpend = '''
</svg>
'''
#ensure at least one per category
if bool(int(force_show_whole_body)):
categories = avatar3dids_helper(theme)['by_category']
for category_name, ids in categories.items():
has_ids_in_category = any([ele in accept_ids for ele in ids])
if not has_ids_in_category:
accept_ids.append(ids[0])
# asseble response
avatar_3d_base_path = get_avatar_attrs(theme, 'path')
if avatar_3d_base_path:
with open(avatar_3d_base_path) as file:
elements = []
tree = ET.parse(file)
for item in tree.getroot():
include_item = item.attrib.get(
'id') in accept_ids or item.tag in tags
if include_item:
elements.append(ET.tostring(item).decode('utf-8'))
output = prepend + "".join(elements) + postpend
tone_maps = get_avatar_attrs(theme, 'tone_maps')
for _type in tone_maps:
base_tone = skinTone if 'hair' not in _type else hairTone
if base_tone:
for _from, to in get_avatar_tone_map(_type,
base_tone).items():
output = output.replace(_from, to)
if request.method == 'POST':
return save_custom_avatar(request, output)
response = HttpResponse(output, content_type='image/svg+xml')
return response
def create_example(xml_file):
#process the xml file
tree = ET.parse(xml_file)
root = tree.getroot()
image_name = root.find('filename').text
file_name = image_name.encode('utf8')
size=root.find('size')
width = int(size[0].text)
height = int(size[1].text)
xmin = []
ymin = []
xmax = []
ymax = []
classes = []
classes_text = []
truncated = []
poses = []
difficult_obj = []
for member in root.findall('object'):
classes_text.append('Pigeon'.encode('utf8'))
xmin.append(float(member[4][0].text) / width)
ymin.append(float(member[4][1].text) / height)
xmax.append(float(member[4][2].text) / width)
ymax.append(float(member[4][3].text) / height)
difficult_obj.append(0)
#if you have more than one classes in dataset you can change the next line
#to read the class from the xml file and change the class label into its
#corresponding integer number, u can use next function structure
'''
def class_text_to_int(row_label):
if row_label == 'Person':
return 1
if row_label == 'car':
return 2
and so on.....
'''
classes.append(1) # i wrote 1 because i have only one class(person)
truncated.append(0)
poses.append('Unspecified'.encode('utf8'))
#read corresponding image
full_path = os.path.join('./images', '{}'.format(image_name)) #provide the path of images directory
with tf.gfile.GFile(full_path, 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = Image.open(encoded_jpg_io)
if image.format != 'JPEG':
raise ValueError('Image format not JPEG')
key = hashlib.sha256(encoded_jpg).hexdigest()
#create TFRecord Example
example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(height),
'image/width': dataset_util.int64_feature(width),
'image/filename': dataset_util.bytes_feature(file_name),
'image/source_id': dataset_util.bytes_feature(file_name),
'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
'image/encoded': dataset_util.bytes_feature(encoded_jpg),
'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
'image/object/difficult': dataset_util.int64_list_feature(difficult_obj),
'image/object/truncated': dataset_util.int64_list_feature(truncated),
'image/object/view': dataset_util.bytes_list_feature(poses),
}))
return example
import json
import os
import xml.etree.ElementTree as ET
def parse_as_polygon(coordinate_string):
point_strings = coordinate_string.split(' ')
coordinate_triples = [point_string.split(',') for point_string in point_strings]
coordinates = [[float(coordinate_triple[0]), float(coordinate_triple[1])] for coordinate_triple in coordinate_triples]
return coordinates
tree = ET.parse('Manhattan_Neighborhoods.kml')
root = tree.getroot()
geojson = {
'type': 'FeatureCollection',
'features': []
}
index = 0
for placemark in root.iter('{http://www.opengis.net/kml/2.2}Placemark'):
name = placemark.find('{http://www.opengis.net/kml/2.2}name').text
coordinates = ''
for coordinate in placemark.iter('{http://www.opengis.net/kml/2.2}coordinates'):
coordinates = coordinate.text
polygon = parse_as_polygon(coordinates)
feature = {
'type': 'Feature',
'id': str(index),
'properties': {
'name': name
editinglist = "yes"
# xbmcgui.Dialog().ok(texture, editinglist,editinglist)
while (editinglist == "yes"):
editinglist = xbmc.getInfoLabel("Skin.String(editinglist)")
# log("still sleeping")
time.sleep(.01)
time.sleep(.01)
playlistfile = xbmc.translatePath( editingthelist )
contents = xbmcvfs.File(playlistfile, 'r')
contents_data = contents.read().decode('utf-8')
xmldata = xmltree.fromstring(contents_data.encode('utf-8'))
for line in xmldata.getiterator():
if line.tag == "name":
newiconname = line.text + ".icon"
# log("found name")
time.sleep(.01)
# newtexture = xbmc.getInfoLabel("Container(50000).ListItem(0).Icon")
# stringnameforicon2 = xbmc.getInfoLabel("Container(50000).ListItem(0).Label2")
# log("getting new texture")
# heading = texture + " old texture"
# line1 = "new texture"
# line2 = stringnameforicon + " old"
# line3 = newiconname + " new"
def write_xml(filename, root):
tree = ET.ElementTree(root)
tree.write(filename)
def CDATA(text=None):
element = ET.Element("![CDATA[")
element.text = text
return element
def auto_ping(url):
sitemap_url = 'http://www.lyyyuna.com/sitemap.xml'
bing_ping = 'http://www.bing.com/webmaster/ping.aspx?siteMap=' + sitemap_url
get = urllib2.urlopen(bing_ping)
result = get.read()
print result
print
## import xml.etree.ElementTree as ET
root = ET.Element("methodCall")
methodname = ET.SubElement(root, "methodName").text = 'weblogUpdates.extendedPing'
params = ET.SubElement(root, "params")
param = ET.SubElement(params, "param")
value = ET.SubElement(param, 'value')
string = ET.SubElement(value, 'string').text = u'lyyyuna 的小花园'
param = ET.SubElement(params, "param")
value = ET.SubElement(param, 'value')
string = ET.SubElement(value, 'string').text = u'http://www.lyyyuna.com/'
param = ET.SubElement(params, "param")
value = ET.SubElement(param, 'value')
string = ET.SubElement(value, 'string').text = url
param = ET.SubElement(params, "param")
value = ET.SubElement(param, 'value')
string = ET.SubElement(value, 'string').text = u'http://www.lyyyuna.com/atom.xml'
# tree = ET.ElementTree(root)
xmlstr = ET.tostring(root, encoding='utf8', method='xml')
# print xmlstr
# print
baidu_pingRPC = 'http://ping.baidu.com/ping/RPC2'
req = urllib2.Request(baidu_pingRPC, xmlstr)
response = urllib2.urlopen(req)
the_page = response.read()
print the_page
print
import xml.etree.ElementTree as ET # Use cElementTree or lxml if too slow
OSM_FILE = "boulder.osm"
SAMPLE_FILE = "sample.osm"
k = 10 # Parameter: take every k-th top level element
def get_element(osm_file, tags=('node', 'way', 'relation')):
"""Yield element if it is the right type of tag
Reference:
http://stackoverflow.com/questions/3095434/inserting-newlines-in-xml-file-generated-via-xml-etree-elementtree-in-python
"""
context = iter(ET.iterparse(osm_file, events=('start', 'end')))
_, root = next(context)
for event, elem in context:
if event == 'end' and elem.tag in tags:
yield elem
root.clear()
with open(SAMPLE_FILE, 'wb') as output:
output.write('<?xml version="1.0" encoding="UTF-8"?>\n')
output.write('<osm>\n ')
# Write every kth top level element
for i, element in enumerate(get_element(OSM_FILE)):
if i % k == 0:
output.write(ET.tostring(element, encoding='utf-8'))
output.write('</osm>')
def parseXmlFiles(xml_path,img_path):
xmllist1 = [i[:-4] for i in os.listdir(xml_path)]
imglist1 = [i[:-4] for i in os.listdir(img_path)]
xmllist = [val+'.xml' for val in xmllist1 if val in imglist1]
xmllist.sort()
for f in xmllist:
if not f.endswith('.xml'):
continue
bndbox = dict()
size = dict()
current_image_id = None
current_category_id = None
file_name = None
size['width'] = None
size['height'] = None
size['depth'] = None
xml_file = os.path.join(xml_path, f)
print(xml_file)
tree = ET.parse(xml_file)
root = tree.getroot() # 抓根结点元素
if root.tag != 'annotation': # 根节点标签
raise Exception('pascal voc xml root element should be annotation, rather than {}'.format(root.tag))
# elem is <folder>, <filename>, <size>, <object>
for elem in root:
current_parent = elem.tag
current_sub = None
object_name = None
# elem.tag, elem.attrib，elem.text
if elem.tag == 'folder':
continue
if elem.tag == 'filename':
file_name = elem.text
if file_name in category_set:
raise Exception('file_name duplicated')
# add img item only after parse <size> tag
elif current_image_id is None and file_name is not None and size['width'] is not None:
if file_name not in image_set:
current_image_id = addImgItem(file_name, size) # 图片信息
print('add image with {} and {}'.format(file_name, size))
else:
raise Exception('duplicated image: {}'.format(file_name))
# subelem is <width>, <height>, <depth>, <name>, <bndbox>
for subelem in elem:
bndbox['xmin'] = None
bndbox['xmax'] = None
bndbox['ymin'] = None
bndbox['ymax'] = None
current_sub = subelem.tag
if current_parent == 'object' and subelem.tag == 'name':
object_name = subelem.text
# if object_name not in category_set:
# current_category_id = addCatItem(object_name)
# else:
# current_category_id = category_set[object_name]
current_category_id = category_set.index(object_name) + 1 # index默认从0开始,但是json文件是从1开始，所以+1
elif current_parent == 'size':
if size[subelem.tag] is not None:
raise Exception('xml structure broken at size tag.')
size[subelem.tag] = int(subelem.text)
# option is <xmin>, <ymin>, <xmax>, <ymax>, when subelem is <bndbox>
for option in subelem:
if current_sub == 'bndbox':
if bndbox[option.tag] is not None:
raise Exception('xml structure corrupted at bndbox tag.')
bndbox[option.tag] = int(option.text)
# only after parse the <object> tag
if bndbox['xmin'] is not None:
if object_name is None:
raise Exception('xml structure broken at bndbox tag')
if current_image_id is None:
raise Exception('xml structure broken at bndbox tag')
if current_category_id is None:
raise Exception('xml structure broken at bndbox tag')
bbox = []
# x
bbox.append(bndbox['xmin'])
# y
bbox.append(bndbox['ymin'])
# w
bbox.append(bndbox['xmax'] - bndbox['xmin'])
# h
bbox.append(bndbox['ymax'] - bndbox['ymin'])
print(
'add annotation with {},{},{},{}'.format(object_name, current_image_id - 1, current_category_id,
bbox))
addAnnoItem(object_name, current_image_id - 1, current_category_id, bbox)
# categories部分
for categoryname in category_set:
addCatItem(categoryname)
difficult = obj.find('difficult').text
cls = obj.find('name').text
if cls not in classes or int(difficult) == 1:
continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))
bb = convert((w,h), b)
out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
xml_path = '/home/ankit.gupta/tool_detect_yolo/dataset/train_annot_folder/'
txt_path = '/home/ankit.gupta/yolo_pytorch/pytorch-0.4-yolov3/data/annotations_txt/'
dataset = '/home/ankit.gupta/yolo_pytorch/pytorch-0.4-yolov3/data/'
split_ratio = 0.2
if not os.path.exists(txt_path):
os.mkdir(txt_path)
annotations_xml = glob.glob(xml_path+'*.xml')
list_file_train = open(dataset + 'train.txt', 'w')
list_file_valid = open(dataset + 'valid.txt', 'w')
sample_factor = int(1/split_ratio)
for i in range(len(annotations_xml)):
tree=ET.parse(annotations_xml[i])
root = tree.getroot()
image_id = root.find('path').text
if (i+1)%sample_factor == 0:
list_file_valid.write(image_id+'\n')
else:
list_file_train.write(image_id+'\n')
convert_annotation(annotations_xml[i], txt_path)
# -*- coding: utf-8 -*-
import urllib
import urllib2
sitemap_url = 'http://www.lyyyuna.com/sitemap.xml'
get = urllib2.urlopen(sitemap_url)
xml_string = get.read()
import sqlite3
import xml.etree.ElementTree as ET
root = ET.fromstring(xml_string)
cx = sqlite3.connect('sitemap.db3')
cur = cx.cursor()
cur.execute('select * from urls')
all_urls = cur.fetchall()
# print all_urls[0][1]
def check_if_exist(url):
for cached_url in all_urls:
ascii_cached_url = cached_url[1].encode('ascii')
# print ascii_cached_url
if url == ascii_cached_url:
return True
return False
def auto_ping(url):
sitemap_url = 'http://www.lyyyuna.com/sitemap.xml'
def rbridge_id_route_map_content_set_origin_origin_incomplete(**kwargs):
"""Auto Generated Code
"""
config = ET.Element("config")
rbridge_id = ET.SubElement(config, "rbridge-id", xmlns="urn:brocade.com:mgmt:brocade-rbridge")
if kwargs.pop('delete_rbridge_id', False) is True:
delete_rbridge_id = config.find('.//*rbridge-id')
delete_rbridge_id.set('operation', 'delete')
rbridge_id_key = ET.SubElement(rbridge_id, "rbridge-id")
rbridge_id_key.text = kwargs.pop('rbridge_id')
if kwargs.pop('delete_rbridge_id', False) is True:
delete_rbridge_id = config.find('.//*rbridge-id')
delete_rbridge_id.set('operation', 'delete')
route_map = ET.SubElement(rbridge_id, "route-map", xmlns="urn:brocade.com:mgmt:brocade-ip-policy")
if kwargs.pop('delete_route_map', False) is True:
delete_route_map = config.find('.//*route-map')
delete_route_map.set('operation', 'delete')
name_key = ET.SubElement(route_map, "name")
name_key.text = kwargs.pop('name')
if kwargs.pop('delete_name', False) is True:
delete_name = config.find('.//*name')
delete_name.set('operation', 'delete')
action_rm_key = ET.SubElement(route_map, "action-rm")
action_rm_key.text = kwargs.pop('action_rm')
if kwargs.pop('delete_action_rm', False) is True:
delete_action_rm = config.find('.//*action-rm')
delete_action_rm.set('operation', 'delete')
instance_key = ET.SubElement(route_map, "instance")
instance_key.text = kwargs.pop('instance')
if kwargs.pop('delete_instance', False) is True:
delete_instance = config.find('.//*instance')
delete_instance.set('operation', 'delete')
content = ET.SubElement(route_map, "content")
if kwargs.pop('delete_content', False) is True:
delete_content = config.find('.//*content')
delete_content.set('operation', 'delete')
set = ET.SubElement(content, "set")
if kwargs.pop('delete_set', False) is True:
delete_set = config.find('.//*set')
delete_set.set('operation', 'delete')
origin = ET.SubElement(set, "origin")
if kwargs.pop('delete_origin', False) is True:
delete_origin = config.find('.//*origin')
delete_origin.set('operation', 'delete')
origin_incomplete = ET.SubElement(origin, "origin-incomplete")
if kwargs.pop('delete_origin_incomplete', False) is True:
delete_origin_incomplete = config.find('.//*origin-incomplete')
delete_origin_incomplete.set('operation', 'delete')
callback = kwargs.pop('callback', _callback)
return callback(config, mgr=kwargs.pop('mgr'))
def test_original_hpcg_parser(datadir):
from akrr.parsers.hpcg_parser import process_appker_output
import xml.etree.ElementTree as ElementTree
from akrr.util import floats_are_close
results = process_appker_output(
appstdout=str(datadir / "original_hpcg" / 'appstdout'),
stdout=str(datadir / "original_hpcg" / 'stdout'),
stderr=str(datadir / "original_hpcg" / 'stderr'),
geninfo=str(datadir / "original_hpcg" / 'gen.info'),
resource_appker_vars={'resource': {'name': 'HPC-Cluster'}}
)
# check resulting xml
xml_out = ElementTree.fromstring(results.replace("'", ""))
params = xml_out.find(".//parameters")
stats = xml_out.find(".//statistics")
assert xml_out.find('./exitStatus/completed').text == "true"
assert len(params.find(".//parameter[ID='App:ExeBinSignature']").find('value').text) > 5
assert len(params.find(".//parameter[ID='RunEnv:Nodes']").find('value').text) > 5
assert params.find(".//parameter[ID='resource']").find('value').text == "HPC-Cluster"
assert params.find(".//parameter[ID='App:Version']").find('value').text == "HPCG-Benchmark version 3.0"
assert params.find(".//parameter[ID='RunEnv:CPU Speed']").find('value').text == "2100.0"
assert params.find(".//parameter[ID='Input:Global Problem Dimensions Nx']").find('value').text == "208"
assert params.find(".//parameter[ID='Input:Global Problem Dimensions Ny']").find('value').text == "208"
assert params.find(".//parameter[ID='Input:Global Problem Dimensions Nz']").find('value').text == "104"
assert params.find(".//parameter[ID='Input:Local Domain Dimensions Nx']").find('value').text == "104"
assert params.find(".//parameter[ID='Input:Local Domain Dimensions Ny']").find('value').text == "104"
assert params.find(".//parameter[ID='Input:Local Domain Dimensions Nz']").find('value').text == "104"
assert params.find(".//parameter[ID='Input:Number of Coarse Grid Levels']").find('value').text == "3"
assert params.find(".//parameter[ID='Input:Distributed Processes']").find('value').text == "4"
assert params.find(".//parameter[ID='Input:Threads per processes']").find('value').text == "1"
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Raw DDOT']").find('value').text), 0.637654)
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Raw MG']").find('value').text), 0.778318)
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Raw SpMV']").find('value').text), 0.91135)
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Raw Total']").find('value').text), 0.80397)
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Raw WAXPBY']").find('value').text), 5.65298)
assert floats_are_close(float(stats.find(
".//statistic[ID='Floating-Point Performance, Total']").find('value').text), 0.773896)
assert floats_are_close(float(stats.find(
".//statistic[ID='Memory Bandwidth, Read']").find('value').text), 4.95353)
assert floats_are_close(float(stats.find(
".//statistic[ID='Memory Bandwidth, Total']").find('value').text), 6.0983)
assert floats_are_close(float(stats.find(
".//statistic[ID='Memory Bandwidth, Write']").find('value').text), 1.14477)
assert floats_are_close(float(stats.find(
".//statistic[ID='Setup Time']").find('value').text), 40.2448)
assert floats_are_close(float(stats.find(
".//statistic[ID='Wall Clock Time']").find('value').text), 385.0)
assert stats.find(".//statistic[ID='App kernel executable exists']").find('value').text == '1'
assert stats.find(".//statistic[ID='App kernel input exists']").find('value').text == '1'
assert stats.find(".//statistic[ID='Network scratch directory accessible']").find('value').text == '1'
assert stats.find(".//statistic[ID='Network scratch directory exists']").find('value').text == '1'
assert stats.find(".//statistic[ID='Task working directory accessible']").find('value').text == '1'
assert stats.find(".//statistic[ID='Task working directory exists']").find('value').text == '1'
assert stats.find(".//statistic[ID='local scratch directory accessible']").find('value').text == '1'
assert stats.find(".//statistic[ID='local scratch directory exists']").find('value').text == '1'
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
import sys
import xml.etree.ElementTree as ET
# Usage:
# ./GenerateAnonymizationProfile.py ~/Subversion/dicom-specification/2017c/part15.xml
if len(sys.argv) != 2:
raise Exception(
'Please provide the path to the part15.xml file from the DICOM standard'
)
with open(sys.argv[1], 'r') as f:
root = ET.fromstring(f.read())
br = '{http://docbook.org/ns/docbook}' # Shorthand variable
LINES = []
def FormatLine(command, name):
indentation = 65
if len(command) > indentation:
raise Exception('Too long command')
line = ' ' + command + (' ' *
(indentation - len(command))) + '// ' + name
LINES.append(line)
def open(self, filename, checkMeshFilePresence=True):
""" parse \a filename and append its content to the current model
"""
Sofa.msg_info("SofaPython.sml","Model: opening file: " + filename)
self.modelDir = os.path.abspath(os.path.dirname(filename))
with open(filename,'r') as f:
# TODO automatic DTD validation could go here, not available in python builtin ElementTree module
modelXml = etree.parse(f).getroot()
if self.name is None and "name" in modelXml.attrib:
self.name = modelXml.attrib["name"]
else:
self.name = os.path.basename(filename)
# units
self.parseUnits(modelXml)
# meshes
for m in modelXml.iter("mesh"):
if not m.find("source") is None:
if m.attrib["id"] in self.meshes:
Sofa.msg_warning("SofaPython.sml","Model: mesh id {0} already defined".format(m.attrib["id"]) )
mesh = Model.Mesh(m)
if not mesh.source is None:
sourceFullPath = os.path.join(self.modelDir,mesh.source)
if os.path.exists(sourceFullPath):
mesh.source=sourceFullPath
elif checkMeshFilePresence :
Sofa.msg_warning("SofaPython.sml","Model: mesh not found: "+mesh.source )
self.meshes[m.attrib["id"]] = mesh
# images
for m in modelXml.iter("image"):
if not m.find("source") is None:
if m.attrib["id"] in self.images:
Sofa.msg_warning("SofaPython.sml","Model: image id {0} already defined".format(m.attrib["id"]) )
image = Model.Image(m)
sourceFullPath = os.path.join(self.modelDir,image.source)
if os.path.exists(sourceFullPath):
image.source=sourceFullPath
else:
Sofa.msg_warning("SofaPython.sml","Model: image not found: "+image.source )
self.images[m.attrib["id"]] = image
# solids
for objXml in modelXml.findall("solid"):
if objXml.attrib["id"] in self.solids:
Sofa.msg_error("SofaPython.sml","Model: solid defined twice, id:" + objXml.attrib["id"])
continue
solid=Model.Solid(objXml)
self.parseMeshes(solid, objXml)
self.parseImages(solid, objXml)
self.solids[solid.id]=solid
# skinning
for objXml in modelXml.findall("solid"):
solid=self.solids[objXml.attrib["id"]]
# TODO: support multiple meshes for skinning (currently only the first mesh is skinned)
for s in objXml.findall("skinning"):
if not s.attrib["solid"] in self.solids:
Sofa.msg_error("SofaPython.sml","Model: skinning for solid {0}: solid {1} is not defined".format(solid.name, s.attrib["solid"]) )
continue
skinning = Model.Skinning()
if not s.attrib["solid"] in self.solids :
Sofa.msg_error("SofaPython.sml","Model: skinning for solid {0}: bone (solid) {1} not defined".format(solid.name, s.attrib["solid"]) )
continue
skinning.solid = self.solids[s.attrib["solid"]]
if not s.attrib["mesh"] in self.meshes :
Sofa.msg_error("SofaPython.sml","Model: skinning for solid {0}: mesh {1} not defined".format(solid.name, s.attrib["mesh"]) )
continue
skinning.mesh = self.meshes[s.attrib["mesh"]]
#TODO: check that this mesh is also part of the solid
if not (s.attrib["group"] in skinning.mesh.group and s.attrib["weight"] in skinning.mesh.group[s.attrib["group"]].data):
Sofa.msg_error("SofaPython.sml","Model: skinning for solid {0}: mesh {1} - group {2} - weight {3} is not defined".format(solid.name, s.attrib["mesh"], s.attrib["group"], s.attrib["weight"]))
continue
skinning.index = skinning.mesh.group[s.attrib["group"]].index
skinning.weight = skinning.mesh.group[s.attrib["group"]].data[s.attrib["weight"]]
solid.skinnings.append(skinning)
# joints
self.parseJointGenerics(modelXml)
# contacts
for c in modelXml.findall("surfaceLink"):
if c.attrib["id"] in self.surfaceLinks:
Sofa.msg_error("SofaPython.sml","Model: surfaceLink defined twice, id:", c.attrib["id"])
continue
surfaceLink = Model.SurfaceLink(c)
surfaces=c.findall("surface")
for i,s in enumerate(surfaces):
# a surfaceLink has at least two surfaces (initialized to None)
if i>= 2:
surfaceLink.surfaces.append( Model.Surface() )
else:
surfaceLink.surfaces[i] = Model.Surface()
if not "solid" in s.attrib: # required
Sofa.msg_error("SofaPython.sml", "Missing solid attribute in surface definition - surfaceLink: "+surfaceLink.id)
continue
if not "mesh" in s.attrib: # required
Sofa.msg_error("SofaPython.sml", "Missing mesh attribute in surface definition - surfaceLink: "+surfaceLink.id)
continue
if s.attrib["solid"] in self.solids:
surfaceLink.surfaces[i].solid = self.solids[s.attrib["solid"]]
else:
Sofa.msg_error("SofaPython.sml","Model: in contact {0}, unknown solid {1} referenced".format(surfaceLink.name, s.attrib["solid"]))
if s.attrib["mesh"] in self.meshes:
surfaceLink.surfaces[i].mesh = self.meshes[s.attrib["mesh"]]
else:
Sofa.msg_error("SofaPython.sml","Model: in contact {0}, unknown mesh {1} referenced".format(surfaceLink.name, s.attrib["mesh"]))
if "group" in s.attrib: # optional
if len(s.attrib["group"]): # discard empty string
surfaceLink.surfaces[i].group = s.attrib["group"]
#if "image" in s.attrib: # optional
# if len(s.attrib["image"]): # discard empty string
# if s.attrib["image"] in self.images:
# reg.surfaces[i].image = self.images[s.attrib["image"]]
self.surfaceLinks[surfaceLink.id]=surfaceLink
def postprocess_translations(reduce_diff_hacks=False):
print('Checking and postprocessing...')
if reduce_diff_hacks:
global _orig_escape_cdata
_orig_escape_cdata = ET._escape_cdata
ET._escape_cdata = escape_cdata
for (filename, filepath) in all_ts_files():
os.rename(filepath, filepath + '.orig')
have_errors = False
for (filename, filepath) in all_ts_files('.orig'):
# pre-fixups to cope with transifex output
parser = ET.XMLParser(
encoding='utf-8'
) # need to override encoding because 'utf8' is not understood only 'utf-8'
with open(filepath + '.orig', 'rb') as f:
data = f.read()
# remove control characters; this must be done over the entire file otherwise the XML parser will fail
data = remove_invalid_characters(data)
tree = ET.parse(io.BytesIO(data), parser=parser)
# iterate over all messages in file
root = tree.getroot()
for context in root.findall('context'):
for message in context.findall('message'):
numerus = message.get('numerus') == 'yes'
source = message.find('source').text
translation_node = message.find('translation')
# pick all numerusforms
if numerus:
translations = [
i.text for i in translation_node.findall('numerusform')
]
else:
translations = [translation_node.text]
for translation in translations:
if translation is None:
continue
errors = []
valid = check_format_specifiers(source, translation,
errors, numerus)
for error in errors:
print('%s: %s' % (filename, error))
if not valid: # set type to unfinished and clear string if invalid
translation_node.clear()
translation_node.set('type', 'unfinished')
have_errors = True
# Remove location tags
for location in message.findall('location'):
message.remove(location)
# Remove entire message if it is an unfinished translation
if translation_node.get('type') == 'unfinished':
context.remove(message)
# check if document is (virtually) empty, and remove it if so
num_messages = 0
for context in root.findall('context'):
for message in context.findall('message'):
num_messages += 1
if num_messages < MIN_NUM_MESSAGES:
print('Removing %s, as it contains only %i messages' %
(filepath, num_messages))
continue
# write fixed-up tree
# if diff reduction requested, replace some XML to 'sanitize' to qt formatting
if reduce_diff_hacks:
out = io.BytesIO()
tree.write(out, encoding='utf-8')
out = out.getvalue()
out = out.replace(b' />', b'/>')
with open(filepath, 'wb') as f:
f.write(out)
else:
tree.write(filepath, encoding='utf-8')
return have_errors
def _openXML(self,xmlPath):
metaTree = ET.parse(xmlPath)
return metaTree.getroot()
# share folder
command_share = "curl -s -X POST -u "
command_share += args.username + ":"
command_share += args.password + " -d "
command_share += "'path=/" + foldername + "&shareType=3&publicUpload=true&permissions=4' '"
command_share += args.url + "/ocs/v2.php/apps/files_sharing/api/v1/shares' -H \"OCS-APIRequest: true\""
if args.verbose:
print(command_share)
resp = subprocess.check_output(command_share, shell=True).decode('utf8').strip('\n')
if args.verbose:
print(resp)
root = ElementTree.fromstring(resp)
for urls in root.iter('url'):
print(foldername + " - " + urls.text)
for ids in root.iter('id'):
# udpate perission
command_perm = "curl -s -o /dev/null -X PUT -u "
command_perm += args.username + ":"
command_perm += args.password + " -d "
command_perm += "'permissions=4' '"
command_perm += args.url + "/ocs/v2.php/apps/files_sharing/api/v1/shares/" + ids.text + "' -H \"OCS-APIRequest: true\""
if args.verbose:
print(command_perm)
else :
serviceurl = 'https://maps.googleapis.com/maps/api/geocode/xml?'
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while True:
address = input('Enter location: ')
if len(address) < 1: break
parms = dict()
parms['address'] = address
if api_key is not False: parms['key'] = api_key
url = serviceurl + urllib.parse.urlencode(parms)
print('Retrieving', url)
uh = urllib.request.urlopen(url, context=ctx)
data = uh.read()
print('Retrieved', len(data), 'characters')
print(data.decode())
tree = ET.fromstring(data)
results = tree.findall('result')
lat = results[0].find('geometry').find('location').find('lat').text
lng = results[0].find('geometry').find('location').find('lng').text
location = results[0].find('formatted_address').text
print('lat', lat, 'lng', lng)
print(location)
self.current_value = None
self.running = True #setting the thread running to true
def run(self):
global gpsd
while gpsp.running:
gpsd.next(
) #this will continue to loop and grab EACH set of gpsd info to clear the buffer
if __name__ == '__main__':
gpsp = GpsPoller() # create the thread
try:
gpsp.start() # start it up
gpx = ET.Element('gpx')
rte = ET.SubElement(gpx, 'rte')
# create a new XML file with the results
mydata = ET.ElementTree(gpx)
mydata.write("testrit.gpx")
while True:
#It may take a second or two to get good data
#print gpsd.fix.latitude,', ',gpsd.fix.longitude,' Time: ',gpsd.utc
#######################
# Getting GPS readings#
#######################
sensor1 = gpsd.fix.latitude
sensor2 = gpsd.fix.longitude
d[key] += 1
else:
d[key] = 1
print(d)
'''
import xml.etree.ElementTree as ET
fileDir = 'D:/EcliplseProjects/'
inFileName = 'test1.xml'
outFileName = inFileName.replace('.xml', '')
#print(outFileName)
tree = ET.parse(fileDir+inFileName)
root = tree.getroot()
print(root, root.tag, root.attrib )
#dummy_root= ET.Element(root.tag)
#print('dummy_root : ', dummy_root )
split_xml = dict()
for child in root:
print(child, child.tag, child.attrib )
att1 = child.attrib['id']
print(att1, type(att1))
if not (att1 in split_xml):
# try:
# tree = ET.parse(interface)
# except ET.ParseError as ex:
# print(ex)
# root = tree.getroot()
# for define in root.iter(xmlns_cd + 'define'):
# name = define.attrib['name']
# COMMONS[name] = define
commands = {}
for interface in contents:
if 'common' in interface.name:
continue
print(interface.name)
try:
tree = ET.parse(interface)
except ET.ParseError as ex:
print(ex)
root = tree.getroot()
for cmd in root.iter(xmlns_cd + 'command'):
name = cmd.attrib['name']
if 'variant' in cmd.attrib \
and (cmd.attrib['variant'].startswith('instance') \
or cmd.attrib['variant']=='assignment'):
# instances need to get resolved
# e.g. note => footnote, endnote
continue
if 'type' in cmd.attrib \
and cmd.attrib['type']=='environment' \
and not name.startswith('start'):
name = 'start'+name
def load_manifest(manifest):
try:
man = ElementTree.parse(manifest).getroot()
except (IOError, ElementTree.ParseError):
man = ElementTree.Element("manifest")
return man
def update_abc_model_xml (modeldir,expt,a,b,c,d,A,B,C,T,SI,vpeak,vinit,uinit):
modelxml = modeldir+'/model.xml';
exptxml = modeldir+'/experiment'+`expt`+'.xml';
# We have to deal with the namespace used in the model.xml file.
ns = {'UL': 'http://www.shef.ac.uk/SpineMLNetworkLayer',
'LL': 'http://www.shef.ac.uk/SpineMLLowLevelNetworkLayer'}
# Parse the model to find the parameters.
import xml.etree.ElementTree as et
et.register_namespace('', "http://www.shef.ac.uk/SpineMLNetworkLayer")
et.register_namespace('LL', "http://www.shef.ac.uk/SpineMLLowLevelNetworkLayer")
tree = et.parse(modelxml)
root = tree.getroot()
updated = 0
for child in root.findall('./*/LL:Neuron/', ns):
nm = child.get('name')
#print 'Model file ', nm;
if nm == 'a':
_a = float(child.find('*').get('value'))
if a != _a:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `a`)
updated = 1
elif nm == 'b':
_b = float(child.find('*').get('value'))
if b != _b:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `b`)
updated = 1
elif nm == 'c':
_c = float(child.find('*').get('value'))
if c != _c:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `c`)
updated = 1
elif nm == 'd':
_d = float(child.find('*').get('value'))
if d != _d:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `d`)
updated = 1
elif nm == 'A':
_A = float(child.find('*').get('value'))
if A != _A:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `A`)
updated = 1
elif nm == 'B':
_B = float(child.find('*').get('value'))
if B != _B:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `B`)
updated = 1
elif nm == 'C':
_C = float(child.find('*').get('value'))
if C != _C:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `C`)
updated = 1
elif nm == 'T':
_T = float(child.find('*').get('value'))
if T != _T:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `T`)
updated = 1
elif nm == 'SI':
_SI = float(child.find('*').get('value'))
if SI != _SI:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `SI`)
updated = 1
elif nm == 'v':
_vinit = float(child.find('*').get('value'))
if vinit != _vinit:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `vinit`)
updated = 1
elif nm == 'u':
_uinit = float(child.find('*').get('value'))
if uinit != _uinit:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `uinit`)
updated = 1
elif nm == 'Vpeak':
_vpeak = float(child.find('*').get('value'))
if vpeak != _vpeak:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `vpeak`)
updated = 1
if (updated > 0):
tree.write(modelxml)
# Parse the expt data to find the currents and any parameter overrides
updated = 0
tree = et.parse(exptxml)
root = tree.getroot()
for child in root.findall('.//UL:Property', ns): # or './*'
nm = child.get('name')
#print 'Expt file ', nm
if nm == 'a':
_a = float(child.find('UL:FixedValue').get('value'))
print '_a: ', _a
if a != _a:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `a`)
updated = 1
elif nm == 'b':
_b = float(child.find('*').get('value'))
if b != _b:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `b`)
updated = 1
elif nm == 'c':
_c = float(child.find('*').get('value'))
if c != _c:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `c`)
updated = 1
elif nm == 'd':
_d = float(child.find('*').get('value'))
if d != _d:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `d`)
updated = 1
elif nm == 'A':
_A = float(child.find('*').get('value'))
if A != _A:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `A`)
updated = 1
elif nm == 'B':
_B = float(child.find('*').get('value'))
if B != _B:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `B`)
updated = 1
elif nm == 'C':
_C = float(child.find('*').get('value'))
if C != _C:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `C`)
updated = 1
elif nm == 'T':
_T = float(child.find('*').get('value'))
if T != _T:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `T`)
updated = 1
elif nm == 'SI':
_SI = float(child.find('*').get('value'))
if SI != _SI:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `SI`)
updated = 1
elif nm == 'v':
_vinit = float(child.find('*').get('value'))
if vinit != _vinit:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `vinit`)
updated = 1
elif nm == 'u':
_uinit = float(child.find('*').get('value'))
if uinit != _uinit:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `uinit`)
updated = 1
elif nm == 'Vpeak':
_vpeak = float(child.find('*').get('value'))
if vpeak != _vpeak:
fixedvalue = child.find('UL:FixedValue', ns)
fixedvalue.set('value', `vpeak`)
updated = 1
if (updated > 0):
tree.write(exptxml)
def import_nmap(self, filename,
args): #domains={}, ips={}, rejects=[] == temp while no db
nFile = filename
try:
tree = ET.parse(nFile)
root = tree.getroot()
hosts = root.findall("host")
except:
print(nFile + " doesn't exist somehow...skipping")
return domains, ips, rejects #needs to be changed for db
tmpNames = []
tmpIPs = {
} #tmpIps = {'127.0.0.1':['domain.com']} -- not really used; decided to just let nslookup take care of IP info
skip = []
for host in hosts:
hostIP = host.find("address").get("addr")
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
for hostname in host.findall("hostnames/hostname"):
hostname = hostname.get("name")
hostname = hostname.lower().replace("www.", "")
reHostname = re.search(
r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}",
hostname) #attempt to not get PTR record
if not reHostname and not args.internal:
created, domain = self.Domain.find_or_create(
domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
elif not reHostname and args.internal:
created, domain = self.Domain.find_or_create(
domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
#else:
# print("IP hostname found? %s" % hostname)
for port in host.findall("ports/port"):
if port.find("state").get("state"):
portState = port.find("state").get("state")
hostPort = port.get("portid")
portProto = port.get("protocol")
created, db_port = self.Port.find_or_create(
port_number=hostPort,
status=portState,
proto=portProto,
ip_address=ip)
if port.find("service") != None:
portName = port.find("service").get("name")
if portName == "http" and hostPort == "443":
portName = "https"
else:
portName = "Unknown"
if created:
db_port.service_name = portName
info = db_port.info
if not info:
info = {}
for script in port.findall(
"script"): #just getting commonName from cert
if script.get("id") == "ssl-cert":
db_port.cert = script.get('output')
cert_domains = self.get_domains_from_cert(
script.get('output'))
for hostname in cert_domains:
hostname = hostname.lower().replace("www.", "")
created, domain = self.Domain.find_or_create(
domain=hostname)
if created:
print("New domain found: %s" % hostname)
elif script.get("id") == "vulners":
print(
"Gathering vuln info for {} : {}/{}\n".format(
hostIP, portProto, hostPort))
self.parseVulners(script.get("output"), db_port)
elif script.get("id") == "banner":
info["banner"] = script.get("output")
elif script.get("id") == "http-headers":
httpHeaders = script.get("output")
httpHeaders = httpHeaders.strip().split("\n")
keepHeaders = self.parseHeaders(httpHeaders)
info["http-headers"] = keepHeaders
elif script.get("id") == "http-auth":
info['http-auth'] = script.get("output")
elif script.get("id") == "http-title":
info['http-title'] = script.get("output")
db_port.info = info
db_port.save()
self.IPAddress.commit()
agg_dataset_names.add(ds_name)
# Come up with which apf data to search from
search_groups = [
"ScienceH5FileStateVectorFields",
"ScienceH5FileRetrievalFields",
"SubsettedL1BFieldNames",
"SubsettedL1BFrameFieldNames",
"SubsettedL1BFieldNames",
"SubsettedDOASFieldNames",
"SubsettedCloudFieldNames",
"SubsettedSndSelFieldNames",
]
for apf_fn, hdf_def_fn in zip(args.apf_list, args.hdf_def_list):
apf_file = ET.parse(apf_fn)
hdf_def_file = ET.parse(hdf_def_fn)
search_fields = []
for grp_name in search_groups:
srch1 = apf_file.find("*[@name='%s']" % grp_name)
if srch1 != None:
search_fields += srch1
else:
srch2 = apf_file.find("*/*[@name='%s']" % grp_name)
if srch2 != None:
search_fields += srch2
else:
print("Could not find group: %s in %s" % (grp_name, apf_fn), file=sys.stderr)
def getHourlyData(self, URL, URLParams, headers):
d = self.openURL(URL, URLParams, headers=headers)
if d is None:
return False
try:
tree = e.parse(d)
except:
return False
#tree = e.parse("/tmp/noaa-fl-2019-06-04-1.xml")
if tree.getroot().tag == 'error':
log.error("*** No hourly information found in response!")
self.lastKnownError = "Retrying hourly data retrieval"
tree.getroot().clear()
del tree
tree = None
return False
# Reset lastKnownError from a previous function call
self.lastKnownError = ""
# We get them in English units need in Metric units
# 2019-06-01: If we send that weather properties we want (qpf=qpf&mint=mint) in request URL NOAA response forgets
# past hours in current day resulting in a forecast requested at the end of the day
# having null/0 qpf forgetting the older values which could had more qpf so we need to process QPF first and
# determine which entries don't have full days with qpf reported (especially current day) then completely skip
# this day for the rest of the weather properties so we don't have a forecast entry with null/0 qpf
# Algorithm allows multiple partial days to be skipped because incomplete but we currently only skip today
# QPF needs to be the first tag parsed to build the skippedDays structure
qpf = self.__parseWeatherTag(tree,
'precipitation',
'liquid',
skippedDays=self.skippedDays,
addToSkippedDays=True)
qpf = convertInchesToMM(qpf)
maxt = self.__parseWeatherTag(tree,
'temperature',
'maximum',
skippedDays=self.skippedDays)
maxt = convertFahrenheitToCelsius(maxt)
mint = self.__parseWeatherTag(
tree,
'temperature',
'minimum',
useStartTimes=False,
skippedDays=self.skippedDays
) # for mint we want the end-time to be saved in DB
mint = convertFahrenheitToCelsius(mint)
temp = self.__parseWeatherTag(tree,
'temperature',
'hourly',
skippedDays=self.skippedDays)
temp = convertFahrenheitToCelsius(temp)
dew = self.__parseWeatherTag(tree,
'temperature',
'dew point',
skippedDays=self.skippedDays)
dew = convertFahrenheitToCelsius(dew)
wind = self.__parseWeatherTag(tree,
'wind-speed',
'sustained',
skippedDays=self.skippedDays)
wind = convertKnotsToMS(wind)
# These are as percentages
pop = self.__parseWeatherTag(tree,
'probability-of-precipitation',
'12 hour',
skippedDays=self.skippedDays)
pop = convertToInt(pop)
humidity = self.__parseWeatherTag(tree,
'humidity',
'relative',
skippedDays=self.skippedDays)
humidity = convertToFloat(humidity)
minHumidity = self.__parseWeatherTag(tree,
'humidity',
'minimum relative',
skippedDays=self.skippedDays)
minHumidity = convertToFloat(minHumidity)
maxHumidity = self.__parseWeatherTag(tree,
'humidity',
'maximum relative',
skippedDays=self.skippedDays)
maxHumidity = convertToFloat(maxHumidity)
if self.parserDebug:
tree.write('noaa-' +
str(rmTimestampToDateAsString(rmCurrentTimestamp())) +
".xml")
tree.getroot().clear()
del tree
tree = None
# Save
self.addValues(RMParser.dataType.MINTEMP, mint)
self.addValues(RMParser.dataType.MAXTEMP, maxt)
self.addValues(RMParser.dataType.TEMPERATURE, temp)
self.addValues(RMParser.dataType.QPF, qpf)
self.addValues(RMParser.dataType.DEWPOINT, dew)
self.addValues(RMParser.dataType.WIND, wind)
self.addValues(RMParser.dataType.POP, pop)
self.addValues(RMParser.dataType.RH, humidity)
self.addValues(RMParser.dataType.MINRH, minHumidity)
self.addValues(RMParser.dataType.MAXRH, maxHumidity)
return True
def AddDischargeDetail(filePath, instrDepManager, disManager):
root = ElementTree.parse(filePath).getroot()
instrument = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Instrument').text
firmware = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Firmware_Version').text
serialNum = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('ADCP_Serial_No').text
software = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Software_Version').text
frequency = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('System_Frequency').text
comments = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Comments').text
numberOfPanels = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Num_Cells').text
adcpDepth = root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('ADCP_Depth').text
# print comments is None
# counter = 0
# for i in root.find('Summary').findall('Station'):
# counter += 1
color = instrDepManager.manager.gui.importedBGColor
instrDepManager.deploymentCmbo = ""
instrDepManager.GetDeploymentCmbo().SetBackgroundColour(color)
instrDepManager.instrumentCmbo = "ADCP"
instrDepManager.GetInstrumentCmbo().SetBackgroundColour(color)
instrDepManager.manufactureCmbo = "TRDI"
instrDepManager.GetManufactureCmbo().SetBackgroundColour(color)
if instrument is not None and instrument != "":
instrDepManager.modelCmbo = instrument.split(" ")[0]
instrDepManager.GetModelCmbo().SetBackgroundColour(color)
if serialNum is not None and serialNum != "":
instrDepManager.serialCmbo = serialNum
instrDepManager.GetSerialCmbo().SetBackgroundColour(color)
# if counter is not None:
# instrDepManager.numOfPanelsScroll = str(counter)
if numberOfPanels is not None and numberOfPanels != "":
instrDepManager.numOfPanelsScroll = numberOfPanels
instrDepManager.GetNumOfPanelsScroll().SetBackgroundColour(color)
if frequency is not None and frequency != "":
instrDepManager.frequencyCmbo = frequency
instrDepManager.GetFrequencyCmbo().SetBackgroundColour(color)
if firmware is not None and firmware != "":
instrDepManager.firmwareCmbo = firmware
instrDepManager.GetFirmwareCmbo().SetBackgroundColour(color)
if software is not None and software != "":
instrDepManager.softwareCtrl = software
instrDepManager.GetSoftwareCtrl().SetBackgroundColour(color)
if comments is not None and comments != "":
disManager.dischRemarksCtrl += '\nSxS measurement notes: ' + comments if disManager.dischRemarksCtrl != "" else 'SxS measurement notes: ' + comments
# instrDepManager.GetDischTxt().SetBackgroundColour(color)
# instrDepManager.GetDischTxt().SetForegroundColour("blue")
try:
float(adcpDepth)
if adcpDepth is not None and adcpDepth != "":
# print adcpDepth
# print len(adcpDepth)
instrDepManager.adcpDepthCtrl = adcpDepth
instrDepManager.GetAdcpDepthCtrl().SetBackgroundColour(color)
except:
pass
# root = ElementTree.parse("C:\\eHSN\\eHSN_Version_1_2_4_ReleasedVersion_Ingest\\SxsMmt\\09CD001_20160120_Good mmnt\\09CD001_20160120.sxs.xml").getroot()
# qtotal = 0
# widthTotal = 0
# areaTotal = 0
# averageVel = 0
# counter = 0
# for i in root.find('Summary').findall('Station'):
# # print i.find('Stn_Q').text
# # print i.find('Width').text
# # print i.find('Area').text
# # print i.find('Avg_V').text
# qtotal += float(i.find('Stn_Q').text)
# widthTotal += float(i.find('Width').text)
# areaTotal += float(i.find('Area').text)
# averageVel += float(i.find('Avg_V').text)
# counter += 1
# print "--------------"
# print counter
# print qtotal
# print widthTotal
# print areaTotal
# print averageVel / (counter - 2)
# print qtotal / areaTotal
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Start_End_Time').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Instrument').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Firmware_Version').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('ADCP_Serial_No').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Software_Version').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('System_Frequency').text
# print root.find('Summary').find('WinRiver_II_Section_by_Section_Summary').find('Comments').text
def _extractSingleXmlFileThread(self, xmlFile):
namespaceprefix = "{http://www.tei-c.org/ns/1.0}"
try:
tree = ET.parse(xmlFile)
root = tree.getroot()
ch_datacolumns = pd.DataFrame(columns=['Filename', 'transcript'])
transcriptionForSpeaker = pd.DataFrame(
columns=ch_datacolumns.columns)
tagsToIgnore = set([
namespaceprefix + tag
for tag in {"gap", "incident", "kinesic", "other"}
])
for utteranceTag in root.iter(namespaceprefix + 'u'):
media = utteranceTag.attrib['start']
filename = media.split('#')[1]
ch_transcript = [""]
for element in utteranceTag:
extractedWord = ""
if (namespaceprefix + "w" == element.tag):
extractedWord = self._extractWordTag(element)
if (namespaceprefix + "pause" == element.tag):
extractedWord = self._extractPauseTag(element)
if (namespaceprefix + "vocal" == element.tag):
extractedWord = self._extractVocalTag(
namespaceprefix, element)
if (namespaceprefix + "del" == element.tag):
extractedWord = self._extractDeletionTag(element)
if (namespaceprefix + "unclear" == element.tag):
extractedWord = self._extractUnclearTag(
namespaceprefix, element)
if (element.tag in tagsToIgnore):
self.logger.debug(
"Found tag {} which is in ignore list, ignoring the whole utterance {}"
.format(element.tag, filename))
break
if (extractedWord):
cleanedWord = self._cleanExtractedWord(extractedWord)
if (cleanedWord):
ch_transcript.append(cleanedWord)
try:
actualTranscript = " ".join(ch_transcript).strip()
if (not actualTranscript
or (self._transcription_pause_tag_symbol()
== actualTranscript)):
self.logger.debug(
"Skipping empty transcription for filename {}".
format(filename))
continue
transcriptionForSpeaker = transcriptionForSpeaker.append(
{
'Filename': filename,
'transcript': actualTranscript
},
ignore_index=True)
transcriptionForSpeaker = self._cleanSpecialCaseWhereTwoSentencesPerFileExist(
transcriptionForSpeaker)
except Exception as e:
self.logger.warn(
"Couldn't append single utterance for filename {}".
format(filename),
exc_info=e)
continue
# writing is just for manual checking
transcriptionForSpeaker.to_csv(os.path.join(
self._getFullFilenameWithoutExtension(xmlFile) +
"_transcript_CH.csv"),
header=True,
index=False,
encoding='utf-8')
return True, xmlFile, transcriptionForSpeaker
except Exception as e:
self.logger.warn(
"Couldn't extract metadata for xml file {}".format(xmlFile),
exc_info=e)
return False, xmlFile, None