def ParseXMLQuest(file):
doc = parseXML(file)
root = doc.documentElement
assert root.tagName == "quest"
assert root.getAttribute("proto") == "xmlquest-1.0"
quest = {
"series": root.getAttribute("series"),
"id": root.getAttribute("id"),
"name": GetText(root.getElementsByTagName("name")),
"text": None,
"html": None,
"solution": [],
"uncheck_verdict": None,
"file": GetText(root.getElementsByTagName("file"))
#"dependencies": [node.getAttribute("quest-id") for node in root.getElementsByTagName("dependency")]
}
if root.getElementsByTagName("checker"):
checker = root.getElementsByTagName("checker")[0]
quest["solution"] = [(GetText([node]), node.getAttribute("case").lower() != "insensitive") for node in checker.getElementsByTagName("solution")]
quest["uncheck_verdict"] = False if checker.getAttribute("strict").lower() == "yes" else None
taskNode = root.getElementsByTagName("task")[0]
if taskNode.getAttribute("type") == "html":
quest["html"] = '<?xml version="1.0" encoding="utf-8" ?>' + ''.join(ch.toxml() for ch in taskNode.childNodes).encode('utf-8')
else:
quest["text"] = GetText([taskNode])
return quest
def __init__(self, config_file):
self.config_file = os.path.abspath(config_file)
self.inputs = {}
dom = parseXML(self.config_file)
s = list(dom_scan(dom, "tool"))
if len(s):
if 'id' in s[0][2]:
self.tool_id = s[0][2]['id']
else:
self.tool_id = None
s = dom_scan(dom, "tool/inputs/param")
for elem, stack, attrs, text in s:
for name, param in self._param_parse(elem):
self.inputs[name] = param
s = dom_scan(dom, "tool/inputs/conditional")
for elem, stack, attrs, text in s:
c = list(dom_scan(elem, "conditional/param"))
if 'name' in attrs:
for p_elem, p_stack, p_attrs, p_text in c:
for name, param in self._param_parse(p_elem, prefix=attrs['name']):
self.inputs[name] = param
self.outputs = {}
s = dom_scan(dom, "tool/outputs/data")
for elem, stack, attrs, text in s:
for name, data in self._data_parse(elem):
self.outputs[name] = data
def run_build(tool_dir, host=None, sudo=False, tool=None, no_cache=False, image_dir=None):
for tool_conf in glob(os.path.join(tool_dir, "*.xml")) + glob(os.path.join(tool_dir, "*", "*.xml")):
logging.info("Scanning: " + tool_conf)
dom = parseXML(tool_conf)
s = dom_scan(dom.childNodes[0], "tool")
if s is not None:
if tool is None or list(s)[0][2]['id'] in tool:
scan = dom_scan(dom.childNodes[0], "tool/requirements/container")
if scan is not None:
for node, prefix, attrs, text in scan:
if 'type' in attrs and attrs['type'] == 'docker':
tag = text
dockerfile = os.path.join(os.path.dirname(tool_conf), "Dockerfile")
if os.path.exists(dockerfile):
call_docker_build(
host = host,
sudo = sudo,
no_cache=no_cache,
tag=tag,
dir=os.path.dirname(tool_conf)
)
if image_dir is not None:
if not os.path.exists(image_dir):
os.mkdir(image_dir)
image_file = os.path.join(image_dir, "docker_" + tag.split(":")[0] + ".tar")
call_docker_save(
host=host,
sudo=sudo,
tag=tag,
output=image_file
)
def parse_zypper_xml(m, cmd, fail_not_found=True, packages=None):
rc, stdout, stderr = m.run_command(cmd, check_rc=False)
dom = parseXML(stdout)
if rc == 104:
# exit code 104 is ZYPPER_EXIT_INF_CAP_NOT_FOUND (no packages found)
if fail_not_found:
errmsg = dom.getElementsByTagName('message')[-1].childNodes[0].data
m.fail_json(msg=errmsg, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
else:
return {}, rc, stdout, stderr
elif rc in [0, 106, 103]:
# zypper exit codes
# 0: success
# 106: signature verification failed
# 103: zypper was upgraded, run same command again
if packages is None:
firstrun = True
packages = {}
solvable_list = dom.getElementsByTagName('solvable')
for solvable in solvable_list:
name = solvable.getAttribute('name')
packages[name] = {}
packages[name]['version'] = solvable.getAttribute('edition')
packages[name]['oldversion'] = solvable.getAttribute('edition-old')
status = solvable.getAttribute('status')
packages[name]['installed'] = status == "installed"
packages[name]['group'] = solvable.parentNode.nodeName
if rc == 103 and firstrun:
# if this was the first run and it failed with 103
# run zypper again with the same command to complete update
return parse_zypper_xml(m, cmd, fail_not_found=fail_not_found, packages=packages)
return packages, rc, stdout, stderr
m.fail_json(msg='Zypper run command failed with return code %s.'%rc, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
def _parse_repos(module):
"""parses the output of zypper -x lr and returns a parse repo dictionary"""
cmd = ['/usr/bin/zypper', '-x', 'lr']
from xml.dom.minidom import parseString as parseXML
rc, stdout, stderr = module.run_command(cmd, check_rc=False)
if rc == 0:
repos = []
dom = parseXML(stdout)
repo_list = dom.getElementsByTagName('repo')
for repo in repo_list:
opts = {}
for o in REPO_OPTS:
opts[o] = repo.getAttribute(o)
opts['url'] = repo.getElementsByTagName('url')[0].firstChild.data
# A repo can be uniquely identified by an alias + url
repos.append(opts)
return repos
# exit code 6 is ZYPPER_EXIT_NO_REPOS (no repositories defined)
elif rc == 6:
return []
else:
d = { 'zypper_exit_code': rc }
if stderr:
d['stderr'] = stderr
if stdout:
d['stdout'] = stdout
module.fail_json(msg='Failed to execute "%s"' % " ".join(cmd), **d)
def get_docker_image(self):
dom = parseXML(self.config_file)
docker_tag = None
scan = dom_scan(dom, "tool/requirements/container")
if scan is not None:
for node, prefix, attrs, text in scan:
if 'type' in attrs and attrs['type'] == 'docker':
docker_tag = text
return docker_tag
def ParseXMLQuest(file):
doc = parseXML(file)
root = doc.documentElement
assert root.tagName == "quest"
proto = root.getAttribute("proto")
assert proto in ["xmlquest-1.0", "xmlquest-2.0"]
if proto == "xmlquest-1.0":
return ParseXMLQuestV1(root)
if proto == "xmlquest-2.0":
return ParseXMLQuestV2(root)
def __init__(self, layer, stackfile=None):
self.layer = layer
stackfile = pathjoin(self.layer.config.dirpath, stackfile)
stack = parseXML(stackfile).firstChild
assert stack.tagName == 'stack', \
'Expecting root element "stack" but got "%s"' % stack.tagName
self.stack = makeStack(stack)
def scan_dir(self, tool_dir):
#scan through directory looking for tool_dir/*/*.xml files and
#attempting to load them
for tool_conf in glob(os.path.join(os.path.abspath(tool_dir), "*.xml")) + glob(os.path.join(os.path.abspath(tool_dir), "*", "*.xml")):
dom = parseXML(tool_conf)
s = list(dom_scan(dom, "tool"))
if len(s):
if 'id' in s[0][2]:
tool_id = s[0][2]['id']
self.config_files[tool_id] = tool_conf
self.tools[tool_id] = GalaxyTool(tool_conf)
def read(self):
#parse file
dom = parseXML(self._f)
self._encoding = dom.encoding
if not dom.doctype.publicId == u"-//Apple//DTD PLIST 1.0//EN":
raise ValueError("The given file is no XML plist file")
#get root
root = dom.documentElement
mainObjectElement = _childElements(root)[0]
return self._visitElement(mainObjectElement)
def tool_dir_scan(tool_dir):
for tool_conf in glob(os.path.join(os.path.abspath(tool_dir), "*.xml")) + glob(os.path.join(os.path.abspath(tool_dir), "*", "*.xml")):
logging.info("Scanning: " + tool_conf)
dom = parseXML(tool_conf)
s = dom_scan(dom.childNodes[0], "tool")
if s is not None:
docker_tag = None
scan = dom_scan(dom.childNodes[0], "tool/requirements/container")
if scan is not None:
for node, prefix, attrs, text in scan:
if 'type' in attrs and attrs['type'] == 'docker':
docker_tag = text
yield list(s)[0][2]['id'], tool_conf, docker_tag
def __init__(self, parameters, titles=None, url=URL):
self._url = url if url.endswith('?') else url + '?'
self._parameters = {}
self._parameters.update(BASIC_PARAMETERS)
self._parameters.update(parameters)
if titles:
self._parameters['titles'] = titles
self.rawdata = self._urlfetch(self._parameters)
if self._parameters['format'] == 'xml':
self.dom = parseXML(self.rawdata)
print 'DOM ready.'
def search(paths):
for path in paths:
for match in find(path, '*.jar', 'pom.xml'):
try:
if os.path.basename(match) == 'pom.xml':
x = parseXML(match)
deps = x.getElementsByTagName('dependency')
for d in deps:
artifact = d.getElementsByTagName('artifactId')[0].childNodes[0].data
version = d.getElementsByTagName('version')
if version:
try:
version = version[0].childNodes[0].data
m = re.match(r'^\$\{(.+)\}$', version)
if m:
lookup = m.group(1)
version = x.getElementsByTagName(lookup)[0].childNodes[0].data
except Exception:
continue
yield match, artifact, version
continue
filename = os.path.splitext(os.path.basename(match))[0].strip()
version = None
try:
zf = zipfile.ZipFile(match, 'r')
try:
f = zf.open('META-INF/MANIFEST.MF')
contents = f.read()
except KeyError:
continue
finally:
zf.close()
if 'Implementation-Version: ' in contents:
version = contents.split('Implementation-Version: ', 1)[1].split('\n', 1)[0].strip()
except zipfile.BadZipfile:
version = filename.split('-', 1)[1]
if version is not None:
if '"' in version:
version = version.split('"', 3)[1]
version = version.split(' ', 1)[0].split('+', 1)[0]
name = filename.split(version, 1)[0].split('_', 1)[0].rstrip('-_.')
yield match, name, version
except Exception:
import traceback
traceback.print_exc()
def _parse_repos(module):
"""parses the output of zypper -x lr and returns a parse repo dictionary"""
cmd = ['/usr/bin/zypper', '-x', 'lr']
repos = []
rc, stdout, stderr = module.run_command(cmd, check_rc=True)
dom = parseXML(stdout)
repo_list = dom.getElementsByTagName('repo')
for repo in repo_list:
opts = {}
for o in REPO_OPTS:
opts[o] = repo.getAttribute(o)
opts['url'] = repo.getElementsByTagName('url')[0].firstChild.data
# A repo can be uniquely identified by an alias + url
repos.append(opts)
return repos
def __init__(self, config_file):
self.config_file = os.path.abspath(config_file)
self.inputs = {}
dom = parseXML(self.config_file)
s = dom_scan(dom.childNodes[0], "tool/inputs/param")
for elem, stack, attrs, text in s:
for name, param in self._param_parse(elem):
self.inputs[name] = param
s = dom_scan(dom.childNodes[0], "tool/inputs/conditional")
for elem, stack, attrs, text in s:
c = list(dom_scan(elem, "conditional/param"))
if 'name' in attrs:
for p_elem, p_stack, p_attrs, p_text in c:
for name, param in self._param_parse(p_elem, prefix=attrs['name']):
self.inputs[name] = param
def request(self, attrs):
attrs.update({'Service': self.service,
'Version': self.api_version,
'AWSAccessKeyId': self.access_key,
'AssociateTag': self.associate_tag,
'Timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ',
time.gmtime())})
query = []
keys = sorted(attrs.keys())
for k in keys:
query.append('%s=%s' % (k, urllib.quote(attrs[k])))
query = "&".join(query)
query += '&Signature=%s' % self._makeSignature(query)
url = '%s?%s' % (self.api_url, query)
return parseXML(urllib2.urlopen(url).read())
def render_cmdline(self, inputs, outputs):
t = None
dom = parseXML(self.config_file)
s = dom_scan(dom, "tool/command")
inter = None
for elem, stack, attrs, text in s:
if elem.attributes.has_key("interpreter"):
inter = elem.attributes['interpreter'].value
t = text
temp = Template(t, searchList=[inputs, outputs], filter=CMDFilter)
out = str(temp)
out = out.replace("\n", " ").strip()
if inter is not None:
res = re.search(r'^([^\s]+)(\s.*)$', out)
print out
spath = os.path.join(self.tool_dir(), res.group(1))
if os.path.exists( spath ):
out = spath + res.group(2)
out = inter + " " + out
return out
def parse_county_coordinates( self, county_id, xml ) : total_boundaries = 0 # число найденных контуров total_points = 0 # число найденных вершин dom = parseXML( xml ) polygons = dom.getElementsByTagName( 'Polygon' ) for polygon in polygons : # Внешние границы графства outer_boundaries = polygon.getElementsByTagName( 'outerBoundaryIs' ) ( boundary_count, point_count ) = self.parse_boundaries( county_id, outer_boundaries, 1 ) total_boundaries += boundary_count total_points += point_count # Границы "вырезов" внутри графства inner_boundaries = polygon.getElementsByTagName( 'innerBoundaryIs' ) ( boundary_count, point_count ) = self.parse_boundaries( county_id, inner_boundaries, 0 ) total_boundaries += boundary_count total_points += point_count # TODO: здесь мы вычисляем центр графства и его площадь return ( total_points, total_boundaries )
def __init__(self, layer, stack=None, stackfile=None):
""" Make a new Composite.Provider.
Arguments:
layer:
The current TileStache.Core.Layer
stack:
A list or dictionary with configuration for the image stack, parsed
by build_stack(). Also acceptable is a URL to a JSON file.
stackfile:
*Deprecated* filename for an XML representation of the image stack.
"""
self.layer = layer
if type(stack) in (str, unicode):
stack = jsonload(urlopen(urljoin(layer.config.dirpath, stack)).read())
if type(stack) in (list, dict):
self.stack = build_stack(stack)
elif stack is None and stackfile:
#
# The stackfile argument is super-deprecated.
#
stackfile = pathjoin(self.layer.config.dirpath, stackfile)
stack = parseXML(stackfile).firstChild
assert stack.tagName == 'stack', \
'Expecting root element "stack" but got "%s"' % stack.tagName
self.stack = makeStack(stack)
else:
raise Exception('Note sure what to do with this stack argument: %s' % repr(stack))
def write_wiki_summary(node_title, node_dir):
"""
Obtain 1 sentence summaries from wikipedia for corresponding wikipedia page for 'node_title'
node_title: title to be used for wikipedia query
node_dir: directory containing content for given node (wiki_summary_file will be written here)
wiki_summary_file: directory containing the node directories
"""
# try to obtain a wiki summary
wiki_ep = 'http://en.wikipedia.org/w/api.php'
urlparams = '?action=query&redirects&prop=extracts&exintro&explaintext&exsectionformat=plain&exsentences=1&format=xml&titles=%s'\
% urllib.quote_plus(node_title)
rquest = urllib2.Request(wiki_ep + urlparams)
xmlresp = parseXML(urllib2.urlopen(rquest))
extxt = xmlresp.getElementsByTagName('extract')
if len(extxt):
summary = extxt[0].firstChild.wholeText.replace('\n',' ')
else:
summary = ''
# cache the wiki summary
temp, tag = os.path.split(node_dir)
content_path, _ = os.path.split(temp)
wiki_summary_file = formats.wiki_summary_file(content_path, tag)
with open(wiki_summary_file, 'w') as wikif:
wikif.write(summary.encode('utf-8'))
def __init__(self, path):
"""Initialize factory."""
self._path = path
# Mainly for testing
if path:
self._data = parseXML(path).documentElement
import sys, os
testfile = sys.argv[1]
whitelist = sys.argv[2]
from xml.dom.minidom import parse as parseXML
allowedErrors = []
fh = open(whitelist, 'r')
for line in fh.readlines():
allowedErrors.append( line.strip() )
fh.close()
document = parseXML( testfile )
suitePasses = True
# look at test suites
for suite in document.childNodes:
# look at testcases
for case in suite.childNodes:
if case.localName == 'testcase':
for child in case.childNodes:
if child.localName == 'error':
if case.getAttribute('name') not in allowedErrors:
print "Got an unallowed error: %s" % case.getAttribute('name')
suitePasses = False
else:
print "Got an allowed error (FIXME): %s" % case.getAttribute('name')
def __init__(self, path):
"""Initialize factory."""
self._path = path
# Mainly for testing
if path:
self._data = parseXML(path).documentElement
def __init__(self, session_ref=None, cache_file=None):
if session_ref and cache_file:
raise Error("can't specify session reference and cache file")
if cache_file == None:
import XenAPI
session = XenAPI.xapi_local()
if not session_ref:
log("No session ref given on command line, logging in.")
session.xenapi.login_with_password("root", "")
else:
session._session = session_ref
try:
inventory = self.__read_xensource_inventory()
assert (inventory.has_key('INSTALLATION_UUID'))
log("host uuid is %s" % inventory['INSTALLATION_UUID'])
host = session.xenapi.host.get_by_uuid(
inventory['INSTALLATION_UUID'])
self.__get_pif_records_from_xapi(session, host)
self.__get_vlan_records_from_xapi(session)
self.__get_bond_records_from_xapi(session)
self.__get_network_records_from_xapi(session)
finally:
if not session_ref:
session.xenapi.session.logout()
else:
log("Loading xapi database cache from %s" % cache_file)
xml = parseXML(root_prefix() + cache_file)
self.__pifs = {}
self.__bonds = {}
self.__vlans = {}
self.__networks = {}
assert (len(xml.childNodes) == 1)
toplevel = xml.childNodes[0]
assert (toplevel.nodeName == "xenserver-network-configuration")
for n in toplevel.childNodes:
if n.nodeName == "#text":
pass
elif n.nodeName == _PIF_XML_TAG:
(ref, rec) = self.__from_xml(n, _PIF_ATTRS)
self.__pifs[ref] = rec
elif n.nodeName == _BOND_XML_TAG:
(ref, rec) = self.__from_xml(n, _BOND_ATTRS)
self.__bonds[ref] = rec
elif n.nodeName == _VLAN_XML_TAG:
(ref, rec) = self.__from_xml(n, _VLAN_ATTRS)
self.__vlans[ref] = rec
elif n.nodeName == _NETWORK_XML_TAG:
(ref, rec) = self.__from_xml(n, _NETWORK_ATTRS)
self.__networks[ref] = rec
else:
raise Error("Unknown XML element %s" % n.nodeName)
"""
import sys
testfile = sys.argv[1]
whitelist = sys.argv[2]
from xml.dom.minidom import parse as parseXML
allowedErrors = []
fh = open(whitelist, 'r')
for line in fh.readlines():
allowedErrors.append(line.strip())
fh.close()
document = parseXML(testfile)
suitePasses = True
# look at test suites
for suite in document.childNodes:
# look at testcases
for case in suite.childNodes:
if case.localName == 'testcase':
for child in case.childNodes:
if child.localName == 'error':
if case.getAttribute('name') not in allowedErrors:
print "Got an unallowed error: %s" % case.getAttribute('name')
suitePasses = False
else:
print "Got an allowed error (FIXME): %s" % case.getAttribute('name')
def __init__(self, session_ref=None, cache_file=None):
if session_ref and cache_file:
raise Error("can't specify session reference and cache file")
if cache_file == None:
import XenAPI
session = XenAPI.xapi_local()
if not session_ref:
log("No session ref given on command line, logging in.")
session.xenapi.login_with_password("root", "")
else:
session._session = session_ref
try:
inventory = self.__read_xensource_inventory()
assert(inventory.has_key('INSTALLATION_UUID'))
log("host uuid is %s" % inventory['INSTALLATION_UUID'])
host = session.xenapi.host.get_by_uuid(inventory['INSTALLATION_UUID'])
self.__get_pif_records_from_xapi(session, host)
self.__get_pool_records_from_xapi(session)
self.__get_tunnel_records_from_xapi(session)
self.__get_vlan_records_from_xapi(session)
self.__get_bond_records_from_xapi(session)
self.__get_network_records_from_xapi(session)
finally:
if not session_ref:
session.xenapi.session.logout()
else:
log("Loading xapi database cache from %s" % cache_file)
xml = parseXML(root_prefix() + cache_file)
self.__pifs = {}
self.__bonds = {}
self.__vlans = {}
self.__pools = {}
self.__tunnels = {}
self.__networks = {}
assert(len(xml.childNodes) == 1)
toplevel = xml.childNodes[0]
assert(toplevel.nodeName == "xenserver-network-configuration")
for n in toplevel.childNodes:
if n.nodeName == "#text":
pass
elif n.nodeName == _PIF_XML_TAG:
(ref,rec) = self.__from_xml(n, _PIF_ATTRS)
self.__pifs[ref] = rec
elif n.nodeName == _BOND_XML_TAG:
(ref,rec) = self.__from_xml(n, _BOND_ATTRS)
self.__bonds[ref] = rec
elif n.nodeName == _VLAN_XML_TAG:
(ref,rec) = self.__from_xml(n, _VLAN_ATTRS)
self.__vlans[ref] = rec
elif n.nodeName == _TUNNEL_XML_TAG:
(ref,rec) = self.__from_xml(n, _TUNNEL_ATTRS)
self.__vlans[ref] = rec
elif n.nodeName == _NETWORK_XML_TAG:
(ref,rec) = self.__from_xml(n, _NETWORK_ATTRS)
self.__networks[ref] = rec
elif n.nodeName == _POOL_XML_TAG:
(ref,rec) = self.__from_xml(n, _POOL_ATTRS)
self.__pools[ref] = rec
else:
raise Error("Unknown XML element %s" % n.nodeName)
self.__get_tunnel_records_from_xapi(session)
except XenAPI.Failure, e:
error,details = e.details
if error == "MESSAGE_METHOD_UNKNOWN" and details == "tunnel.get_all":
pass
self.__get_vlan_records_from_xapi(session)
self.__get_bond_records_from_xapi(session)
self.__get_network_records_from_xapi(session)
finally:
if not session_ref:
session.xenapi.session.logout()
else:
log("Loading xapi database cache from %s" % cache_file)
xml = parseXML(root_prefix() + cache_file)
self.__pifs = {}
self.__bonds = {}
self.__vlans = {}
self.__tunnels = {}
self.__networks = {}
assert(len(xml.childNodes) == 1)
toplevel = xml.childNodes[0]
assert(toplevel.nodeName == "xenserver-network-configuration")
for n in toplevel.childNodes:
if n.nodeName == "#text":
pass
self.__get_tunnel_records_from_xapi(session)
except XenAPI.Failure, e:
error, details = e.details
if error == "MESSAGE_METHOD_UNKNOWN" and details == "tunnel.get_all":
pass
self.__get_vlan_records_from_xapi(session)
self.__get_bond_records_from_xapi(session)
self.__get_network_records_from_xapi(session)
finally:
if not session_ref:
session.xenapi.session.logout()
else:
log("Loading xapi database cache from %s" % cache_file)
xml = parseXML(root_prefix() + cache_file)
self.__pifs = {}
self.__bonds = {}
self.__vlans = {}
self.__tunnels = {}
self.__networks = {}
assert (len(xml.childNodes) == 1)
toplevel = xml.childNodes[0]
assert (toplevel.nodeName == "xenserver-network-configuration")
for n in toplevel.childNodes:
if n.nodeName == "#text":
pass
