def get_snap_info_by_volname(volname, mnode=None):
"""Parse the output of 'gluster snapshot info' command
for the given volume.
Args:
volname (str): snapshot name
Kwargs:
mnode (str): Node on which command has to be executed.
If None, defaults to nodes[0].
Returns:
NoneType: None if command execution fails, parse errors.
list: list of dicts on success.
Examples:
>>> get_snap_info_by_volname('testvol',
mnode = 'abc.lab.eng.xyz.com')
{'originVolume': {'snapCount': '1', 'name': 'testvol',
'snapRemaining': '255'}, 'count': '1', 'snapshots':
[{'description': 'This is next snap1', 'uuid':
'dcf0cd31-c0db-47ad-92ec-f72af2d7b385', 'volCount': '1',
'snapVolume': {'status': 'Stopped', 'name':
'49c290d6e8b74205adb3cce1206b5bc5'}, 'createTime':
'2016-04-07 12:03:11', 'name': 'next_snap1'}]}
"""
if mnode is None:
mnode = tc.servers[0]
cmd = "gluster snapshot info volume %s --xml" % volname
ret, out, _ = tc.run(mnode, cmd, verbose=False)
if ret != 0:
tc.logger.error("Failed to execute 'snapshot info' on node %s. "
"Hence failed to get the snapshot info.", mnode)
return None
try:
root = etree.XML(out)
except etree.ParseError:
tc.logger.error("Failed to parse the gluster snapshot "
"info xml output.")
return None
snap_vol_info = {}
for snap in root.findall("snapInfo"):
for element in snap.getchildren():
if element.tag == "originVolume":
info = {}
for elmt in element.getchildren():
info[elmt.tag] = elmt.text
snap_vol_info[element.tag] = info
else:
snap_vol_info[element.tag] = element.text
snap_info_list = []
for snap in root.findall("snapInfo/snapshots/snapshot"):
snap_info = {}
for element in snap.getchildren():
if element.tag == "snapVolume":
info = {}
for elmt in element.getchildren():
if elmt.tag == "originVolume":
info["originVolume"] = {}
for el in elmt.getchildren():
info[elmt.tag][el.tag] = el.text
else:
info[elmt.tag] = elmt.text
snap_info[element.tag] = info
else:
snap_info[element.tag] = element.text
snap_info_list.append(snap_info)
snap_vol_info["snapshots"] = snap_info_list
return snap_vol_info
def render(self, unused_node, unused_handler):
"""Receive a node and return a node."""
return cElementTree.XML('[Unimplemented custom tag]')
def setUp(self) -> None:
"""Initialize some variables"""
self.xml_string = """
<company>
<staff>
<operations_department>
<employees>
<team>Marketing</team>
<location name="head office" address="Kampala, Uganda" />
<bio first_name="John" last_name="Doe">John Doe</bio>
<bio first_name="Jane" last_name="Doe">Jane Doe</bio>
<bio first_name="Peter" last_name="Doe">Peter Doe</bio>
</employees>
<employees>
<team>Customer Service</team>
<location name="Kampala branch" address="Kampala, Uganda" />
<bio first_name="Mary" last_name="Doe">Mary Doe</bio>
<bio first_name="Harry" last_name="Doe">Harry Doe</bio>
<bio first_name="Paul" last_name="Doe">Paul Doe</bio>
</employees>
</operations_department>
</staff>
</company>
"""
self.expected_xml_output: Element = ElementTree.XML(self.xml_string)
self.expected_output = {
'operations_department': {
'employees': [
[
{
'team': 'Marketing',
'location': {
'name': 'head office',
'address': 'Kampala, Uganda'
},
'first_name': 'John',
'last_name': 'Doe',
'_value': 'John Doe'
},
{
'team': 'Marketing',
'location': {
'name': 'head office',
'address': 'Kampala, Uganda'
},
'first_name': 'Jane',
'last_name': 'Doe',
'_value': 'Jane Doe'
},
{
'team': 'Marketing',
'location': {
'name': 'head office',
'address': 'Kampala, Uganda'
},
'first_name': 'Peter',
'last_name': 'Doe',
'_value': 'Peter Doe'
},
],
[{
'team': 'Customer Service',
'location': {
'name': 'Kampala branch',
'address': 'Kampala, Uganda'
},
'first_name': 'Mary',
'last_name': 'Doe',
'_value': 'Mary Doe'
}, {
'team': 'Customer Service',
'location': {
'name': 'Kampala branch',
'address': 'Kampala, Uganda'
},
'first_name': 'Harry',
'last_name': 'Doe',
'_value': 'Harry Doe'
}, {
'team': 'Customer Service',
'location': {
'name': 'Kampala branch',
'address': 'Kampala, Uganda'
},
'first_name': 'Paul',
'last_name': 'Doe',
'_value': 'Paul Doe'
}],
]
}
}
round(simulation.lambda_start * 1e9, 10),
round(simulation.lambda_end * 1e9, 10),
simulation.resolution
# simulation.resolution,
# simulation.accuracy
)
#%% load parameters from XML files
# can pass an XML file with these parameters, as an argument to python on the command line
PCells_params = []
if (len(sys.argv) > 1):
filepath = sys.argv[1]
print('Loading XML file: %s' % filepath)
with open(filepath, 'r') as file:
d = etree_to_dict(ET.XML(file.read()))['PCells']
if d == None:
print('Error: no PCells found in the XML file')
sys.exit('Error: no PCells found in the XML file')
elif type(d['PCell']) == list:
PCells_params = d['PCell']
elif len(d) == 1:
PCells_params = [d['PCell']]
else:
print('Error: problem with XML file')
sys.exit('Error: problem with XML file')
IDs = []
if (
len(sys.argv) > 2
): # argument is one component to simulate; otherwise simulate all of them.
def fromstring(cls, string, encoding="utf-8"):
return cls(ETree.XML(string))
def xml2dict(self, xml_text):
""" Extract dict with XML text passed
"""
parent_element = ElementTree.XML(xml_text)
result_data = XmlDictConfig(parent_element)
return result_data
return d
parser = argparse.ArgumentParser(
description='Generate a failure report from JUnitXML')
parser.add_argument('filename',
help='JUnit XML file to parse (produced by pytest)')
args = parser.parse_args()
try:
data = open(args.filename, 'r').read()
except FileNotFoundError:
print(f'Could not open JUnitXML file [{args.filename}]')
sys.exit(-1)
xml = ET.XML(data)
json = etree_to_dict(xml)
tests = json['testsuites']['testsuite']['testcase']
for test in tests:
if 'failure' in test:
message = test['failure']['@message']
text = test['failure']['#text']
filename = test['@file']
test_name = test['@name']
first_error = []
for line in text.split('\n'):
if line.startswith('_ _ _ _'):
def newFromXML(cls, raw):
elem = etree.XML(raw)
data = dict([(e.tag, e.text) for e in elem.getiterator() \
if e.tag not in ['oembed']])
return cls.createLoad(data)
def _xml2dict(xml):
"""Convert XML string to dict"""
return Util._xmlET2dict(ET.XML(xml))
def get_activity_from_tcx(tcx):
tree = et.XML(tcx)
for element in tree.iter():
if element.tag == "{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Name":
return "%s" % (element.text.replace(' ', '_'))
def fetch(self, xml=None, sequence=None, **kwargs):
"""Get Blast record from url or file.
:arg sequence: an object with an associated sequence string
or a sequence string itself
:type sequence: :class:`.Atomic`, :class:`.Sequence`, str
:arg xml: blast search results in XML format or an XML file that
contains the results or a filename for saving the results or None
:type xml: str
:arg timeout: amount of time until the query times out in seconds
default value is 120
:type timeout: int
"""
if self.isSuccess:
LOGGER.warn("The record already exists so not further search is performed")
return True
if sequence is None:
sequence = self._sequence
if xml is None:
xml = self._xml
import xml.etree.cElementTree as ET
have_xml = False
filename = None
if xml is not None:
if len(xml) < 100:
# xml likely contains a filename
if os.path.isfile(xml):
# read the contents
try:
xml = ET.parse(xml)
root = xml.getroot()
have_xml = True
except:
raise ValueError('could not parse xml from xml file')
else:
# xml contains a filename for writing
filename = xml
else:
try:
if isinstance(xml, list):
root = ET.fromstringlist(xml)
elif isinstance(xml, str):
root = ET.fromstring(xml)
except:
raise ValueError('xml is not a filename and does not look like'
' a valid XML string')
else:
have_xml = True
if have_xml is False:
# we still need to run a blast
headers = {'User-agent': 'ProDy'}
query = [('DATABASE', 'pdb'), ('ENTREZ_QUERY', '(none)'),
('PROGRAM', 'blastp'),]
expect = float(kwargs.pop('expect', 10e-10))
if expect <= 0:
raise ValueError('expect must be a positive number')
query.append(('EXPECT', expect))
hitlist_size = int(kwargs.pop('hitlist_size', 250))
if hitlist_size <= 0:
raise ValueError('expect must be a positive integer')
query.append(('HITLIST_SIZE', hitlist_size))
query.append(('QUERY', sequence))
query.append(('CMD', 'Put'))
sleep = float(kwargs.pop('sleep', 2))
timeout = float(kwargs.pop('timeout', self._timeout))
self._timeout = timeout
try:
import urllib.parse
urlencode = lambda data: bytes(urllib.parse.urlencode(data), 'utf-8')
except ImportError:
from urllib import urlencode
url = 'https://blast.ncbi.nlm.nih.gov/Blast.cgi'
data = urlencode(query)
LOGGER.timeit('_prody_blast')
LOGGER.info('Blast searching NCBI PDB database for "{0}..."'
.format(sequence[:5]))
handle = openURL(url, data=data, headers=headers)
html = handle.read()
index = html.find(b'RID =')
if index == -1:
raise Exception('NCBI did not return expected response.')
else:
last = html.find(b'\n', index)
rid = html[index + len('RID ='):last].strip()
query = [('ALIGNMENTS', 500), ('DESCRIPTIONS', 500),
('FORMAT_TYPE', 'XML'), ('RID', rid), ('CMD', 'Get')]
data = urlencode(query)
while True:
LOGGER.sleep(int(sleep), 'to reconnect to NCBI for search results.')
LOGGER.write('Connecting to NCBI for search results...')
handle = openURL(url, data=data, headers=headers)
results = handle.read()
index = results.find(b'Status=')
LOGGER.clear()
if index < 0:
break
last = results.index(b'\n', index)
status = results[index+len('Status='):last].strip()
if status.upper() == b'READY':
break
sleep = int(sleep * 1.5)
if LOGGER.timing('_prody_blast') > timeout:
LOGGER.warn('Blast search time out.')
return False
LOGGER.clear()
LOGGER.report('Blast search completed in %.1fs.', '_prody_blast')
root = ET.XML(results)
try:
ext_xml = filename.lower().endswith('.xml')
except AttributeError:
pass
else:
if not ext_xml:
filename += '.xml'
out = open(filename, 'w')
if PY3K:
out.write(results.decode())
else:
out.write(results)
out.close()
LOGGER.info('Results are saved as {0}.'.format(repr(filename)))
root = dictElement(root, 'BlastOutput_')
if root['db'] != 'pdb':
raise ValueError('blast search database in xml must be "pdb"')
if root['program'] != 'blastp':
raise ValueError('blast search program in xml must be "blastp"')
self._param = dictElement(root['param'][0], 'Parameters_')
query_len = int(root['query-len'])
if sequence and len(sequence) != query_len:
raise ValueError('query-len and the length of the sequence do not '
'match, xml data may not be for given sequence')
hits = []
for iteration in root['iterations']:
for hit in dictElement(iteration, 'Iteration_')['hits']:
hit = dictElement(hit, 'Hit_')
data = dictElement(hit['hsps'][0], 'Hsp_')
for key in ['align-len', 'gaps', 'hit-frame', 'hit-from',
'hit-to', 'identity', 'positive', 'query-frame',
'query-from', 'query-to']:
data[key] = int(data[key])
data['query-len'] = query_len
for key in ['evalue', 'bit-score', 'score']:
data[key] = float(data[key])
p_identity = 100.0 * data['identity'] / (data['query-to'] -
data['query-from'] + 1)
data['percent_identity'] = p_identity
p_overlap = (100.0 * (data['align-len'] - data['gaps']) /
query_len)
data['percent_coverage'] = p_overlap
for item in (hit['id'] + hit['def']).split('>gi'):
head, title = item.split(None, 1)
head = head.split('|')
pdb_id = head[-2].lower()
chain_id = head[-1][:1]
pdbch = dict(data)
pdbch['pdb_id'] = pdb_id
pdbch['chain_id'] = chain_id
pdbch['title'] = (head[-1][1:] + title).strip()
hits.append((p_identity, p_overlap, pdbch))
hits.sort(key=lambda hit: hit[0], reverse=True)
self._hits = hits
return True
def xml_to_dict(xml_context):
from xml.etree import cElementTree as ElementTree
_root = ElementTree.XML(xml_context)
_xml_dict = XmlDictConfig(_root)
return _xml_dict
def get_data_from_xml(file_path):
with open(file_path, "r") as fin:
context = fin.read()
t = ET.XML(context)
return etree_to_dict(t)
def searchPfam(query, **kwargs):
"""Returns Pfam search results in a dictionary. Matching Pfam accession
as keys will map to evalue, alignment start and end residue positions.
:arg query: UniProt ID, PDB identifier, a protein sequence, or a sequence
file. Sequence queries must not contain without gaps and must be at
least 16 characters long
:type query: str
:arg timeout: timeout for blocking connection attempt in seconds, default
is 60
:type timeout: int
*query* can also be a PDB identifier, e.g. ``'1mkp'`` or ``'1mkpA'`` with
chain identifier. UniProt ID of the specified chain, or the first
protein chain will be used for searching the Pfam database."""
if isfile(query):
from prody.sequence import MSAFile
try:
seq = next(MSAFile(query))
except:
with openFile(query) as inp:
seq = ''.join(inp.read().split())
else:
seq = seq[0][1]
if not seq.isalpha():
raise ValueError('could not parse a sequence without gaps from ' +
query)
else:
seq = ''.join(query.split())
import xml.etree.cElementTree as ET
LOGGER.timeit('_pfam')
timeout = int(kwargs.get('timeout', 60))
if len(seq) >= MINSEQLEN:
if not seq.isalpha():
raise ValueError(repr(seq) + ' is not a valid sequence')
fseq = '>Seq\n' + seq
parameters = { 'hmmdb' : 'pfam', 'seq': fseq }
enc_params = urllib.urlencode(parameters).encode('utf-8')
request = urllib2.Request('https://www.ebi.ac.uk/Tools/hmmer/search/hmmscan', enc_params)
results_url = urllib2.urlopen(request).geturl()
#res_params = { 'output' : 'xml' }
res_params = { 'format' : 'tsv' }
enc_res_params = urllib.urlencode(res_params)
#modified_res_url = results_url + '?' + enc_res_params
modified_res_url = results_url.replace('results','download') + '?' + enc_res_params
result_request = urllib2.Request(modified_res_url)
# url = ( urllib2.urlopen(request).geturl() + '?output=xml')
LOGGER.debug('Submitted Pfam search for sequence "{0}...".'
.format(seq[:MINSEQLEN]))
try:
#xml = urllib2.urlopen(result_request).read()
tsv = urllib2.urlopen(result_request).read()
# openURL(url, timeout=timeout).read()
except:
raise ValueError('No matching Pfam domains were found.')
# try:
# root = ET.XML(xml)
# except Exception as err:
# raise ValueError('failed to parse results XML, check URL: ' + modified_res_url)
matches = {}
#for child in root[0]:
#if child.tag == 'hits':
# accession = child.get('acc')
# pfam_id = accession.split('.')[0]
# matches[pfam_id]={}
# matches[pfam_id]['accession']=accession
# matches[pfam_id]['class']='Domain'
# matches[pfam_id]['id']=child.get('name')
# matches[pfam_id]['locations']={}
# matches[pfam_id]['locations']['ali_end']=child[0].get('alisqto')
# matches[pfam_id]['locations']['ali_start']=child[0].get('alisqfrom')
# matches[pfam_id]['locations']['bitscore']=child[0].get('bitscore')
# matches[pfam_id]['locations']['end']=child[0].get('alisqto')
# matches[pfam_id]['locations']['evalue']=child.get('evalue')
# matches[pfam_id]['locations']['evidence']='hmmer v3.0'
# matches[pfam_id]['locations']['hmm_end']=child[0].get('alihmmto')
# matches[pfam_id]['locations']['hmm_start']=child[0].get('alihmmfrom')
# matches[pfam_id]['locations']['significant']=child[0].get('significant')
# matches[pfam_id]['locations']['start']=child[0].get('alisqfrom')
# matches[pfam_id]['type']='Pfam-A'
# return matches
if PY3K:
tsv = tsv.decode()
lines = tsv.split('\n')
keys = lines[0].split('\t')
root = {}
for i, line in enumerate(lines[1:-1]):
root[i] = {}
for j, key in enumerate(keys):
root[i][key] = line.split('\t')[j]
for child in root.values():
accession = child['Family Accession']
pfam_id = accession.split('.')[0]
matches[pfam_id]={}
matches[pfam_id]['accession'] = accession
matches[pfam_id]['class'] = 'Domain'
matches[pfam_id]['id'] = child['Family id']
matches[pfam_id]['locations'] = {}
matches[pfam_id]['locations']['ali_end'] = child['Ali. End']
matches[pfam_id]['locations']['ali_start'] = child['Ali. Start']
matches[pfam_id]['locations']['bitscore'] = child['Bit Score']
matches[pfam_id]['locations']['end'] = child['Env. End']
matches[pfam_id]['locations']['cond_evalue'] = child['Cond. E-value']
matches[pfam_id]['locations']['ind_evalue'] = child['Ind. E-value']
matches[pfam_id]['locations']['evidence'] = 'hmmer v3.0'
matches[pfam_id]['locations']['hmm_end'] = child['Model End']
matches[pfam_id]['locations']['hmm_start'] = child['Model Start']
#matches[pfam_id]['locations']['significant'] = child['significant']
matches[pfam_id]['locations']['start'] = child['Env. Start']
matches[pfam_id]['type'] = 'Pfam-A'
return matches
else:
if len(seq) <= 5:
idcode = None
from prody import parsePDBHeader
try:
polymers = parsePDBHeader(seq[:4], 'polymers')
except Exception as err:
LOGGER.warn('failed to parse header for {0} ({1})'
.format(seq[:4], str(err)))
else:
chid = seq[4:].upper()
for poly in polymers:
if chid and poly.chid != chid:
continue
for dbref in poly.dbrefs:
if dbref.database != 'UniProt':
continue
idcode = dbref.idcode
accession = dbref.accession
LOGGER.info('UniProt ID code {0} for {1} chain '
'{2} will be used.'
.format(idcode, seq[:4], poly.chid))
break
if idcode is not None:
break
if idcode is None:
LOGGER.warn('A UniProt ID code for PDB {0} could not be '
'parsed.'.format(repr(seq)))
url = prefix + 'protein/' + seq + '?output=xml'
else:
url = prefix + 'protein/' + idcode + '?output=xml'
else:
url = prefix + 'protein/' + seq + '?output=xml'
LOGGER.debug('Retrieving Pfam search results: ' + url)
xml = None
while LOGGER.timing('_pfam') < timeout:
try:
xml = openURL(url, timeout=timeout).read()
except Exception:
pass
else:
if xml not in ['PEND','RUN']:
break
if not xml:
raise IOError('Pfam search timed out or failed to parse results '
'XML, check URL: ' + url)
else:
LOGGER.report('Pfam search completed in %.2fs.', '_pfam')
if xml.find(b'There was a system error on your last request.') > 0:
LOGGER.warn('No Pfam matches found for: ' + seq)
return None
elif xml.find(b'No valid UniProt accession or ID') > 0:
try:
url = prefix + 'protein/' + accession + '?output=xml'
LOGGER.debug('Retrieving Pfam search results: ' + url)
xml = openURL(url, timeout=timeout).read()
except:
try:
ag = parsePDB(seq, subset='ca')
ag_seq = ag.getSequence()
return searchPfam(ag_seq)
except:
raise ValueError('No valid UniProt accession or ID for: ' + seq)
if xml.find(b'No valid UniProt accession or ID') > 0:
try:
url = 'https://uniprot.org/uniprot/' + accession + '.xml'
xml = openURL(url, timeout=timeout).read()
root = ET.XML(xml)
accession = root[0][0].text
url = prefix + 'protein/' + accession + '?output=xml'
LOGGER.debug('Retrieving Pfam search results: ' + url)
xml = openURL(url, timeout=timeout).read()
except:
raise ValueError('No valid UniProt accession or ID for: ' + seq)
try:
root = ET.XML(xml)
except Exception as err:
raise ValueError('failed to parse results XML, check URL: ' + url)
if len(seq) >= MINSEQLEN:
try:
xml_matches = root[0][0][0][0]
except IndexError:
raise ValueError('failed to parse results XML, check URL: ' + url)
else:
key = '{' + prefix + '}'
results = dictElement(root[0], key)
try:
xml_matches = results['matches']
except KeyError:
raise ValueError('failed to parse results XML, check URL: ' + url)
matches = dict()
for child in xml_matches:
try:
accession = child.attrib['accession'][:7]
except KeyError:
raise ValueError('failed to parse results XML, check URL: ' + url)
if not re.search('^P(F|B)[0-9]{5}$', accession):
raise ValueError('{0} does not match pfam accession'
' format'.format(accession))
match = matches.setdefault(accession, dict(child.items()))
locations = match.setdefault('locations', [])
for loc in child:
locations.append(dict(loc.items()))
if len(seq) < MINSEQLEN:
query = 'Query ' + repr(query)
else:
query = 'Query sequence'
if matches:
LOGGER.info(query + ' matched {0} Pfam families.'.format(len(matches)))
else:
LOGGER.info(query + ' did not match any Pfam families.')
return matches
def rebalance_stop_and_get_status(mnode, volname):
"""Parse the output of 'gluster vol rebalance stop' command
for the given volume
Args:
mnode (str): Node on which command has to be executed.
volname (str): volume name
Returns:
NoneType: None if command execution fails, parse errors.
dict: dict on success. rebalance status will be
in dict format
Examples:
>>> rebalance_stop_and_get_status('abc.xyz.com', testvol)
{'node': [{'files': '0', 'status': '3', 'lookups': '0', 'skipped': '0',
'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00', 'id':
'11336017-9561-4e88-9ac3-a94d4b403340', 'statusStr': 'completed',
'size': '0'}, {'files': '0', 'status': '1', 'lookups': '0', 'skipped':
'0', 'nodeName': '10.70.47.16', 'failures': '0', 'runtime': '0.00',
'id': 'a2b88b10-eba2-4f97-add2-8dc37df08b27', 'statusStr':
'in progress', 'size': '0'}, {'files': '0', 'status': '3',
'lookups': '0', 'skipped': '0', 'nodeName': '10.70.47.152',
'failures': '0', 'runtime': '0.00', 'id':
'b15b8337-9f8e-4ec3-8bdb-200d6a67ae12', 'statusStr': 'completed',
'size': '0'}, {'files': '0', 'status': '3', 'lookups': '0', 'skipped':
'0', 'nodeName': '10.70.46.52', 'failures': '0', 'runtime': '0.00',
'id': '77dc299a-32f7-43d8-9977-7345a344c398', 'statusStr': 'completed',
'size': '0'}], 'task-id': 'a16f99d1-e165-40e7-9960-30508506529b',
'aggregate': {'files': '0', 'status': '1', 'lookups': '0', 'skipped':
'0', 'failures': '0', 'runtime': '0.00', 'statusStr': 'in progress',
'size': '0'}, 'nodeCount': '4', 'op': '3'}
"""
cmd = "gluster volume rebalance %s stop --xml" % volname
ret, out, _ = g.run(mnode, cmd)
if ret != 0:
g.log.error(
"Failed to execute 'rebalance stop' on node %s. "
"Hence failed to parse the rebalance status.", mnode)
return None
try:
root = etree.XML(out)
except etree.ParseError:
g.log.error("Failed to parse gluster rebalance stop xml output.")
return None
rebal_status = {}
rebal_status["node"] = []
for info in root.findall("volRebalance"):
for element in info.getchildren():
if element.tag == "node":
status_info = {}
for elmt in element.getchildren():
status_info[elmt.tag] = elmt.text
rebal_status[element.tag].append(status_info)
elif element.tag == "aggregate":
status_info = {}
for elmt in element.getchildren():
status_info[elmt.tag] = elmt.text
rebal_status[element.tag] = status_info
else:
rebal_status[element.tag] = element.text
return rebal_status
if t.text:
text = t.text.strip()
if children or t.attrib:
if text:
d[t.tag]['#text'] = text
else:
d[t.tag] = text
return d
if __name__ == '__main__':
with open('enip-info3.xml', 'r') as f:
eip_info = f.read()
from xml.etree import cElementTree as ET
devices = list()
e = ET.XML(eip_info)
d = etree_to_dict(e)
for device in d['nmaprun']['host']:
if device['status']['@state'] == 'up':
if 'port' in device['ports']:
output = dict()
if type(device['address']) == list:
addresses = device['address']
else:
addresses = [device['address']]
for addr in addresses:
if addr['@addrtype'] == 'ipv4':
output['ip'] = addr['@addr']
ports_list = list()
if type(device['ports']['port']) == list:
ports = device['ports']['port']
def get_remove_brick_status(mnode, volname, bricks_list):
"""Parse the output of 'gluster vol remove-brick status' command
for the given volume
Args:
mnode (str): Node on which command has to be executed.
volname (str): volume name
bricks_list (list): List of bricks participating in
remove-brick operation
Returns:
NoneType: None if command execution fails, parse errors.
dict: dict on success. rebalance status will be
in dict format
Examples:
>>> get_remove_brick_status('abc.lab.eng.xyz.com', testvol, bricklist)
{'node': [{'files': '0', 'status': '3', 'lookups': '0', 'skipped': '0'
, 'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00','id'
: '6662bdcd-4602-4f2b-ac1a-75e6c85e780c', 'statusStr':
'completed', 'size': '0'}], 'task-id': '6a135147-b202-4e69-
b48c-b1c6408b9d24', 'aggregate': {'files': '0', 'status': '3',
'lookups': '0', 'skipped': '0', 'failures': '0', 'runtime':
'0.00', 'statusStr': 'completed', 'size': '0'}, 'nodeCount'
: '3'}
"""
cmd = ("gluster volume remove-brick %s %s status --xml" %
(volname, ' '.join(bricks_list)))
ret, out, _ = g.run(mnode, cmd)
if ret != 0:
g.log.error("Failed to execute 'remove-brick status' on node %s",
mnode)
return None
try:
root = etree.XML(out)
except etree.ParseError:
g.log.error(
"Failed to parse the remove-brick status"
"xml output on volume %s", volname)
return None
remove_brick_status = {}
remove_brick_status["node"] = []
for info in root.findall("volRemoveBrick"):
for element in info.getchildren():
if element.tag == "node":
status_info = {}
for elmt in element.getchildren():
status_info[elmt.tag] = elmt.text
remove_brick_status[element.tag].append(status_info)
elif element.tag == "aggregate":
status_info = {}
for elmt in element.getchildren():
status_info[elmt.tag] = elmt.text
remove_brick_status[element.tag] = status_info
else:
remove_brick_status[element.tag] = element.text
return remove_brick_status
def run(self):
# Define our components entry point
while True:
# for each document waiting on our input port
for doc in self.receive_all('in'):
try:
data = doc.get('data')
mime = doc.get_meta('mimetype')
# if there is no data, move on to the next doc
if data is None:
continue
# if this is not a xml file, move on to the next doc
if mime != 'application/xml':
continue
# solr xml starts with an add tag, if this does not start
# with an add tag, move to the next doc
xml = ET.XML(data)
if xml.tag != 'add':
log.debug('Not a Solr XML document')
continue
# create a new packet for each document in the xml
for sdoc in xml.getchildren():
if sdoc.tag != 'doc':
log.warn('Unexpected tag %s, expecting doc' % \
sdoc.tag)
continue
packet = Packet()
# each field in the xml becomes a field in the packet
for sfield in sdoc.getchildren():
if sfield.tag != 'field':
log.warn('Unexpected tag %s, expecting field' \
% sfield.tag)
continue
fieldname = sfield.get('name', None)
fieldval = sfield.text
if fieldname is not None:
if not isinstance(fieldname, str):
fieldname = fieldname.decode('utf-8')
if not isinstance(fieldval, str):
fieldval = fieldval.decode('utf-8')
packet.append(fieldname, fieldval)
# send out packet along
self.send('out', packet)
except Exception as e:
log.error('Component Failed: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
#log.error(traceback.print_exc())
# yield the CPU, allowing another component to run
self.yield_ctrl()
def passtimes_stopplace(self, pubsub, town, stoparea, querydatetime=None):
if querydatetime is not None:
try:
querydatetime = dateutil.parser.parse(querydatetime)
except:
return false
else:
querydatetime = datetime.today()
if querydatetime.hour <= 4:
operatingday = querydatetime.date() - timedelta(days=1)
fromtime = datetime.combine(date(1970, 1, 2), querydatetime.time())
else:
operatingday = querydatetime.date()
fromtime = datetime.combine(date(1970, 1, 1), querydatetime.time())
node_arguments = {'town': town, 'stoparea': stoparea }
items = pubsub.stanza.Items()
items['node'] = '/' + self.name + '/postalregion/%(town)s/stoparea/%(stoparea)s/passtimes' % node_arguments
cursor = self.connection.cursor()
cursor.execute(KV1_SQL_STOPPLACE, { 'weekday': 2 ** ((querydatetime.weekday() + 1) % 7), # In SQL Monday is 1, our bitstring wants Sunday at 0.
'operatingday': operatingday,
'from': fromtime,
'stopplace': stoparea,
'maxresults': 10 } )
services = set([])
allrows = cursor.fetchall()
for dataownercode, lineplanningnumber, journeynumber, localservicelevelcode, userstopordernumber, userstopcode, linepublicnumber, linedirection, destinationcode, destinationname50, destinationdetail24, istimingstop, passtime, first, last, wheelchairaccessible in allrows:
services.add((dataownercode, lineplanningnumber))
actual = {}
for service in services:
self.punctuality_client.send(serializer.dumps([service[0], service[1], userstopcode]))
actual.update(serializer.loads(self.punctuality_client.recv()))
for dataownercode, lineplanningnumber, journeynumber, localservicelevelcode, userstopordernumber, userstopcode, linepublicnumber, linedirection, destinationcode, destinationname50, destinationdetail24, istimingstop, passtime, first, last, wheelchairaccessible in allrows:
index = dataownercode + '_' + lineplanningnumber + '_' + str(journeynumber)
if first:
journeystoptype = 'FIRST'
elif last:
journeystoptype = 'LAST'
else:
journeystoptype = 'INTERMEDIATE'
lastupdatetimestamp = datetime.today()
if destinationdetail24 is None:
destinationdetail24 = ''
tripstopstatus = 'UNKNOWN'
expectedarrivaltime = passtime
expecteddeparturetime = passtime
targetarrivaltime = passtime
targetdeparturetime = passtime
if index in actual:
# TODO: some super duper prediction here!
prediction = timedelta(seconds = actual[index]['punctuality'])
if actual[index]['punctuality'] > 0 and actual[index]['hastimingstop']:
expecteddeparturetime = passtime
else:
expecteddeparturetime += prediction
expectedarrivaltime += prediction
tripstopstatus = 'DRIVING'
expectedarrivaltime = self.renderpasstime(expectedarrivaltime)
expecteddeparturetime = self.renderpasstime(expecteddeparturetime)
targetarrivaltime = self.renderpasstime(targetarrivaltime)
targetdeparturetime = self.renderpasstime(targetdeparturetime)
timingpointdataownercode = 'OPENOV'
timingpointcode = userstopcode
sidecode = '-'
fortifyordernumber = 0
arguments = {'dataownercode': dataownercode,
'operationdate': operatingday,
'lineplanningnumber': lineplanningnumber,
'linepublicnumber': linepublicnumber,
'journeynumber': journeynumber,
'fortifyordernumber': fortifyordernumber,
'userstopordernumber': userstopordernumber,
'userstopcode': userstopcode,
'localservicelevelcode': localservicelevelcode,
'linedirection': linedirection,
'lastupdatetimestamp': lastupdatetimestamp,
'destinationcode': destinationcode,
'destinationname50': destinationname50,
'destinationdetail24': destinationdetail24,
'istimingpoint': str(istimingstop).lower(),
'expectedarrivaltime': expectedarrivaltime,
'targetarrivaltime': targetarrivaltime,
'expecteddeparturetime': expecteddeparturetime,
'targetdeparturetime': targetdeparturetime,
'tripstopstatus': tripstopstatus,
'sidecode': sidecode,
'wheelchairaccessible': str(wheelchairaccessible).lower(),
'timingpointdataownercode': timingpointdataownercode,
'timingpointcode': timingpointcode,
'journeystoptype': journeystoptype}
item = pubsub.stanza.Item()
item['id'] = '%(dataownercode)s_%(lineplanningnumber)s_%(journeynumber)s_%(fortifyordernumber)s' % arguments
item['payload'] = ET.XML(KV8_DATEDPASSTIME % arguments)
items.append(item)
return items
def __rrc_sr_callback(self, msg):
# deal with RRC OTA
# cell_id = self.get_analyzer('TrackCellInfoAnalyzer').get_cur_cell_id()
# if cell_id != self.cell_id :
# self.cell_id = cell_id
# self.__clear_counters()
if msg.type_id == "LTE_RRC_OTA_Packet":
log_item = msg.data.decode()
log_item_dict = dict(log_item)
if 'Msg' in log_item_dict:
log_xml = ET.XML(log_item_dict['Msg'])
for field in log_xml.iter('field'):
if field.get(
'name'
) == "lte-rrc.rrcConnectionSetupComplete_element":
# self.__clear_kpi()
if self.cause:
self.kpi_measurements['success_number'][
self.cause] += 1
self.cause = None
self.store_kpi(
"KPI_Accessibility_RRC_SUC",
self.kpi_measurements['total_number'],
log_item_dict['timestamp'])
# upload_dict = dict((k, self.kpi_measurements[k]) for k in ('success_number', 'total_number'))
# self.upload_kpi('KPI.Accessibility.RRC_SR', upload_dict, log_item_dict['timestamp'])
# self.upload_kpi('KPI.Accessibility.RRC_SR', upload_dict)
# self.log_debug(str(upload_dict))
# self.broadcast_info('RRC_SR', upload_dict)
# print self.local_query_kpi('KPI_Accessibility_RRC_SR_MO_DATA', msg.timestamp)
# Upload RRC latency
# delta = (log_item_dict['timestamp'] - self.rrc_req_timestamp).total_seconds()
# upload_dict = {'latency': delta}
# self.upload_kpi("KPI.Accessibility.RRC_LATENCY", upload_dict)
# self.log_debug(str(upload_dict))
elif field.get(
'name') == "lte-rrc.rrcConnectionRequest_element":
self.rrc_req_timestamp = log_item_dict['timestamp']
elif field.get('name') == 'lte-rrc.establishmentCause':
# self.__clear_kpi()
if field.get('show') == '4':
self.cause = 'MO_DATA'
self.kpi_measurements['total_number'][
self.cause] += 1
elif field.get('show') == '3':
self.cause = 'MO_SIGNAL'
self.kpi_measurements['total_number'][
self.cause] += 1
elif field.get('show') == '2':
self.cause = 'MT_ACCESS'
self.kpi_measurements['total_number'][
self.cause] += 1
elif field.get('show') == '1':
self.cause = 'HIGH_PRIORITY_ACCESS'
self.kpi_measurements['total_number'][
self.cause] += 1
elif field.get('show') == '0':
self.cause = 'EMERGENCY'
self.kpi_measurements['total_number'][
self.cause] += 1
else:
self.cause = 'UNKNOWN'
self.kpi_measurements['total_number'][
self.cause] += 1
#FIXME: MobileInsight crashes after reporting this warning
self.log_warning(
"Unknown lte-rrc.establishmentCause: " +
str(field.get('showname')))
self.store_kpi("KPI_Accessibility_RRC_REQ",
self.kpi_measurements['total_number'],
log_item_dict['timestamp'])
elif field.get('name') == 'lte-rrc.releaseCause':
# TODO: check if this release is abnormal
# self.__clear_kpi()
queue_length = self.get_analyzer(
'UlMacLatencyAnalyzer').queue_length
if queue_length != 0:
# Check cause loadBalancingTAUrequired, other, cs-FallbackHighPriority-v1020, rrc-Suspend-v1320
if field.get('show') == '0': # Cause: Other
self.kpi_measurements['release_number'][
'LB_TAU'] += 1
elif field.get('show') == '1': # Cause: Other
self.kpi_measurements['release_number'][
'OTHER'] += 1
elif field.get('show') == '2': # Cause: Other
self.kpi_measurements['release_number'][
'CSFB'] += 1
elif field.get('show') == '3': # Cause: Other
self.kpi_measurements['release_number'][
'SUSPEND'] += 1
else:
#FIXME: MobileInsight crashes after reporting this warning
self.kpi_measurements['release_number'][
'UNKNOWN'] += 1
self.log_warning(
"Unknown lte-rrc.releaseCause: " +
field.get('showname'))
self.store_kpi(
"KPI_Retainability_RRC_AB_REL",
self.kpi_measurements['release_number'],
log_item_dict['timestamp'])
# upload_dict = {'total_number': sum(self.kpi_measurements['total_number'].values()),
# 'release_number': self.kpi_measurements['release_number']}
# self.upload_kpi('KPI.Retainability.RRC_AB_REL', upload_dict, log_item_dict['timestamp'])
# self.upload_kpi('KPI.Retainability.RRC_AB_REL', upload_dict)
# self.log_info(str(upload_dict))
# self.broadcast_info('RRC_SR', upload_dict)
return 0
def shade(self, stroke):
stroke_to_paths = "".join(self.pathgen(stroke, self.style, self.h, self.split_at_invisible, self.stroke_color_mode)).split("\n")
# convert to actual XML. Empty strokes are empty strings; they are ignored.
self.elements.extend(et.XML(elem) for elem in stroke_to_paths if elem) # if len(elem.strip()) > len(self.path))
def xml_to_dict(xml):
root = ElementTree.XML(xml)
return XmlDictConfig(root)
def render(self, unused_arg):
return cElementTree.XML(
'<Complex><Child>Text</Child></Complex>')
def make_xml(text):
xml = ET.XML('<xml>%s</xml>' % text)
return xml
def get_season_nzbs(name, url_data, season):
"""
Split a season NZB into episodes
:param name: NZB name
:param url_data: URL to get data from
:param season: Season to check
:return: dict of (episode files, xml matches)
"""
# TODO: clean up these regex'es, comment them, and make them all raw strings
regex_string = {
# Match the xmlns in an nzb
# Example: nzbElement.getchildren()[1].tag == '{http://www.newzbin.com/DTD/2003/nzb}file'
# regex match returns 'http://www.newzbin.com/DTD/2003/nzb'
'nzb_xmlns': r"{(http://[\w_\./]+nzb)}file",
'scene_name':
'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)', # pylint: disable=anomalous-backslash-in-string
'episode': '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+', # pylint: disable=anomalous-backslash-in-string
}
try:
show_xml = ETree.ElementTree(ETree.XML(url_data))
except SyntaxError:
logger.log(u"Unable to parse the XML of " + name +
", not splitting it", logger.ERROR) # pylint: disable=no-member
return {}, ''
nzb_element = show_xml.getroot()
scene_name_match = re.search(regex_string['scene_name'] % season, name,
re.I)
if scene_name_match:
show_name = scene_name_match.groups()[0]
else: # Make sure we aren't missing valid results after changing name_parser and the quality detection
# Most of these will likely be invalid shows
logger.log(u"Unable to parse " + name + " into a scene name.",
logger.DEBUG) # pylint: disable=no-member
return {}, ''
regex = '(' + re.escape(show_name) + regex_string['episode'] % season + ')'
regex = regex.replace(' ', '.')
ep_files = {}
xmlns = None
for cur_file in nzb_element.getchildren():
xmlns_match = re.match(regex_string['nzb_xmlns'], cur_file.tag)
if not xmlns_match:
continue
else:
xmlns = xmlns_match.group(1)
match = re.search(regex, cur_file.get("subject"), re.I)
if not match:
# regex couldn't match cur_file.get("subject")
continue
cur_ep = match.group(1)
if cur_ep not in ep_files:
ep_files[cur_ep] = [cur_file]
else:
ep_files[cur_ep].append(cur_file)
# TODO: Decide what to do if we found multiple valid xmlns strings, should we only return the last???
return ep_files, xmlns
def _preparse_internal_list(cls, decoded_list):
lst = []
type_id = ""
try:
# for i in range(len(decoded_list)):
i = 0
while i < len(decoded_list):
field_name, val, type_str = decoded_list[i]
if field_name == "type_id":
type_id = val
if type_str.startswith("raw_msg/"):
msg_type = type_str[len("raw_msg/"):]
decoded = cls._decode_msg(msg_type, val)
xmls = [
decoded,
]
if msg_type == "RRC_DL_BCCH_BCH":
sib_types = cls._preparse_internal_list.wcdma_sib_types
try:
# xml = ET.fromstring(decoded)
xml = ET.XML(decoded)
except Exception as e:
print "Unsupported RRC_DL_BCCH_BCH"
xx = cls._wrap_decoded_xml(xmls)
lst.append(("Unsupported", xx, "msg"))
return lst
sibs = xml.findall(
".//field[@name='rrc.CompleteSIBshort_element']")
if sibs:
# deal with a list of complete SIBs
for complete_sib in sibs:
field = complete_sib.find(
"field[@name='rrc.sib_Type']")
if field is None:
continue
sib_id = int(field.get("show"))
sib_name = field.get("showname")
field = complete_sib.find(
"field[@name='rrc.sib_Data_variable']")
if field is None:
continue
sib_msg = binascii.a2b_hex(field.get("value"))
if sib_id in sib_types:
decoded = cls._decode_msg(
sib_types[sib_id], sib_msg)
xmls.append(decoded)
# print sib_types[sib_id]
else:
print "(MI)Unknown RRC SIB Type: %d" % sib_id
else:
# deal with a segmented SIB
sib_segment = xml.find(
".//field[@name='rrc.firstSegment_element']")
if sib_segment is None:
sib_segment = xml.find(
".//field[@name='rrc.subsequentSegment_element']"
)
if sib_segment is None:
sib_segment = xml.find(
".//field[@name='rrc.lastSegmentShort_element']"
)
if sib_segment is not None:
field = sib_segment.find(
"field[@name='rrc.sib_Type']")
sib_id = int(field.get("show"))
# Zengwen: need to use log it back
# print "RRC SIB Segment(type: %d) not handled"
# % sib_id
xx = cls._wrap_decoded_xml(xmls)
lst.append((field_name, xx, "msg"))
else:
lst.append(decoded_list[i])
i = i + 1
return lst, type_id
except Exception as e:
print len(decoded_list)
print decoded_list
i = 0
while i < len(decoded_list):
print i
print decoded_list[i]
i += 1
import traceback
raise RuntimeError(str(traceback.format_exc()))
<filter id='MyFilter' filterUnits='objectBoundingBox' x='0' y='0' width='1' height='1'>
<feGaussianBlur in='SourceAlpha' stdDeviation='4%' result='blur'/>
<feOffset in='blur' dx='4%' dy='4%' result='offsetBlur'/>
<feSpecularLighting in='blur' surfaceScale='5' specularConstant='.75'
specularExponent='20' lighting-color='#bbbbbb' result='specOut'>
<fePointLight x='-5000%' y='-10000%' z='20000%'/>
</feSpecularLighting>
<feComposite in='specOut' in2='SourceAlpha' operator='in' result='specOut'/>
<feComposite in='SourceGraphic' in2='specOut' operator='arithmetic'
k1='0' k2='1' k3='1' k4='0'/>
</filter>
</defs>
"""
tree, xmlid = ET.XMLID(f.getvalue())
# insert the filter definition in the svg dom tree.
tree.insert(0, ET.XML(filter_def))
for i, pie_name in enumerate(labels):
pie = xmlid[pie_name]
pie.set("filter", 'url(#MyFilter)')
shadow = xmlid[pie_name + "_shadow"]
shadow.set("filter", 'url(#dropshadow)')
fn = "svg_filter_pie.svg"
print("Saving '%s'" % fn)
ET.ElementTree(tree).write(fn)
def _parse_internal_list_old(cls, out_type, decoded_list):
"""
DEPRECIATED. DO NOT CALL THIS FUNCTION since it is very slow.
"""
if out_type == "dict":
output_d = dict()
elif out_type == "list":
output_lst = []
elif out_type.startswith("xml/"):
tag_name = out_type[len("xml/"):]
output_xml = ET.Element(tag_name)
# for field_name, val, type_str in decoded_list:
i = 0
while i < len(decoded_list):
# print "debug decoded_list: ",str(decoded_list[i])
field_name, val, type_str = decoded_list[i]
if not type_str: # default type
xx = val
elif type_str == "msg":
xx = val
elif type_str == "dict":
if out_type.startswith("xml/"):
xx = cls._parse_internal_list("xml/dict", val)
else:
xx = cls._parse_internal_list("dict", val)
elif type_str == "list":
if out_type.startswith("xml/"):
xx = cls._parse_internal_list("xml/list", val)
else:
xx = cls._parse_internal_list("list", val)
if out_type == "dict":
output_d[field_name] = xx
elif out_type == "list":
output_lst.append(xx)
elif out_type.startswith("xml/"):
# Create tag
if out_type == "xml/list":
sub_tag = ET.SubElement(output_xml, "item")
elif out_type == "xml/dict":
sub_tag = ET.SubElement(output_xml, "pair",
{"key": field_name})
if not type_str:
xx = str(xx)
sub_tag.text = xx
else:
if type_str == "msg":
# xx = ET.fromstring(xx)
xx = ET.XML(xx)
sub_tag.set("type", "list")
else:
sub_tag.set("type", type_str)
sub_tag.append(xx)
i += 1
if out_type == "dict":
return output_d
elif out_type == "list":
return tuple(output_lst)
elif out_type.startswith("xml/"):
return output_xml
def fetchPDBLigand(cci, filename=None):
"""Fetch PDB ligand data from PDB_ for chemical component *cci*.
*cci* may be 3-letter chemical component identifier or a valid XML
filename. If *filename* is given, XML file will be saved with that name.
If you query ligand data frequently, you may configure ProDy to save XML
files in your computer. Set ``ligand_xml_save`` option **True**, i.e.
``confProDy(ligand_xml_save=True)``. Compressed XML files will be save
to ProDy package folder, e.g. :file:`/home/user/.prody/pdbligands`. Each
file is around 5Kb when compressed.
This function is compatible with PDBx/PDBML v 4.0.
Ligand data is returned in a dictionary. Ligand coordinate atom data with
*model* and *ideal* coordinate sets are also stored in this dictionary.
Note that this dictionary will contain data that is present in the XML
file and all Ligand Expo XML files do not contain every possible data
field. So, it may be better if you use :meth:`dict.get` instead of
indexing the dictionary, e.g. to retrieve formula weight (or relative
molar mass) of the chemical component use ``data.get('formula_weight')``
instead of ``data['formula_weight']`` to avoid exceptions when this data
field is not found in the XML file. URL and/or path of the XML file are
returned in the dictionary with keys ``url`` and ``path``, respectively.
Following example downloads data for ligand STI (a.k.a. Gleevec and
Imatinib) and calculates RMSD between model (X-ray structure 1IEP) and
ideal (energy minimized) coordinate sets:
.. ipython:: python
from prody import *
ligand_data = fetchPDBLigand('STI')
ligand_data['model_coordinates_db_code']
ligand_model = ligand_data['model']
ligand_ideal = ligand_data['ideal']
transformation = superpose(ligand_ideal.noh, ligand_model.noh)
calcRMSD(ligand_ideal.noh, ligand_model.noh)"""
if not isinstance(cci, str):
raise TypeError('cci must be a string')
if isfile(cci):
inp = openFile(cci)
xml = inp.read()
inp.close()
url = None
path = cci
cci = splitext(splitext(split(cci)[1])[0])[0].upper()
elif len(cci) > 4 or not cci.isalnum():
raise ValueError('cci must be 3-letters long and alphanumeric or '
'a valid filename')
else:
xml = None
cci = cci.upper()
if SETTINGS.get('ligand_xml_save'):
folder = join(getPackagePath(), 'pdbligands')
if not isdir(folder):
makePath(folder)
xmlgz = path = join(folder, cci + '.xml.gz')
if isfile(xmlgz):
with openFile(xmlgz) as inp:
xml = inp.read()
else:
path = None
#url = ('http://ligand-expo.rcsb.org/reports/{0[0]}/{0}/{0}'
# '.xml'.format(cci.upper()))
url = 'http://www.pdb.org/pdb/files/ligand/{0}.xml'.format(cci.upper())
if not xml:
#'http://www.pdb.org/pdb/files/ligand/{0}.xml'
try:
inp = openURL(url)
except IOError:
raise IOError(
'XML file for ligand {0} is not found online'.format(cci))
else:
xml = inp.read()
inp.close()
if filename:
out = openFile(filename, mode='w', folder=folder)
out.write(xml)
out.close()
if SETTINGS.get('ligand_xml_save'):
with openFile(xmlgz, 'w') as out:
out.write(xml)
import xml.etree.cElementTree as ET
root = ET.XML(xml)
if (root.get('{http://www.w3.org/2001/XMLSchema-instance}'
'schemaLocation') !=
'http://pdbml.pdb.org/schema/pdbx-v40.xsd pdbx-v40.xsd'):
LOGGER.warn('XML is not in PDBx/PDBML v 4.0 format, resulting '
'dictionary may not contain all data fields')
ns = root.tag[:root.tag.rfind('}') + 1]
len_ns = len(ns)
dict_ = {'url': url, 'path': path}
for child in list(root.find(ns + 'chem_compCategory')[0]):
tag = child.tag[len_ns:]
if tag.startswith('pdbx_'):
tag = tag[5:]
dict_[tag] = child.text
dict_['formula_weight'] = float(dict_.get('formula_weight'))
identifiers_and_descriptors = []
results = root.find(ns + 'pdbx_chem_comp_identifierCategory')
if results:
identifiers_and_descriptors.extend(results)
results = root.find(ns + 'pdbx_chem_comp_descriptorCategory')
if results:
identifiers_and_descriptors.extend(results)
for child in identifiers_and_descriptors:
program = child.get('program').replace(' ', '_')
type_ = child.get('type').replace(' ', '_')
dict_[program + '_' + type_] = child[0].text
dict_[program + '_version'] = child.get('program_version')
dict_['audits'] = [
(audit.get('action_type'), audit.get('date'))
for audit in list(root.find(ns + 'pdbx_chem_comp_auditCategory'))
]
atoms = list(root.find(ns + 'chem_comp_atomCategory'))
n_atoms = len(atoms)
ideal_coords = np.zeros((n_atoms, 3))
model_coords = np.zeros((n_atoms, 3))
atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
elements = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['element'].dtype)
resnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['resname'].dtype)
charges = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
resnums = np.ones(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
alternate_atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
leaving_atom_flags = np.zeros(n_atoms, np.bool)
aromatic_flags = np.zeros(n_atoms, np.bool)
stereo_configs = np.zeros(n_atoms, np.bool)
ordinals = np.zeros(n_atoms, int)
name2index = {}
for i, atom in enumerate(atoms):
data = dict([(child.tag[len_ns:], child.text) for child in list(atom)])
name = data.get('pdbx_component_atom_id', 'X')
name2index[name] = i
atomnames[i] = name
elements[i] = data.get('type_symbol', 'X')
resnames[i] = data.get('pdbx_component_comp_id', 'UNK')
charges[i] = float(data.get('charge', 0))
alternate_atomnames[i] = data.get('alt_atom_id', 'X')
leaving_atom_flags[i] = data.get('pdbx_leaving_atom_flag') == 'Y'
aromatic_flags[i] = data.get('pdbx_atomatic_flag') == 'Y'
stereo_configs[i] = data.get('pdbx_stereo_config') == 'Y'
ordinals[i] = int(data.get('pdbx_ordinal', 0))
model_coords[i, 0] = float(data.get('model_Cartn_x', 0))
model_coords[i, 1] = float(data.get('model_Cartn_y', 0))
model_coords[i, 2] = float(data.get('model_Cartn_z', 0))
ideal_coords[i, 0] = float(data.get('pdbx_model_Cartn_x_ideal', 0))
ideal_coords[i, 1] = float(data.get('pdbx_model_Cartn_y_ideal', 0))
ideal_coords[i, 2] = float(data.get('pdbx_model_Cartn_z_ideal', 0))
pdbid = dict_.get('model_coordinates_db_code')
if pdbid:
model = AtomGroup(cci + ' model ({0})'.format(pdbid))
else:
model = AtomGroup(cci + ' model')
model.setCoords(model_coords)
model.setNames(atomnames)
model.setResnames(resnames)
model.setResnums(resnums)
model.setElements(elements)
model.setCharges(charges)
model.setFlags('leaving_atom_flags', leaving_atom_flags)
model.setFlags('aromatic_flags', aromatic_flags)
model.setFlags('stereo_configs', stereo_configs)
model.setData('ordinals', ordinals)
model.setData('alternate_atomnames', alternate_atomnames)
dict_['model'] = model
ideal = model.copy()
ideal.setTitle(cci + ' ideal')
ideal.setCoords(ideal_coords)
dict_['ideal'] = ideal
bonds = []
warned = set()
for bond in list(root.find(ns + 'chem_comp_bondCategory') or bonds):
name_1 = bond.get('atom_id_1')
name_2 = bond.get('atom_id_2')
try:
bonds.append((name2index[name_1], name2index[name_2]))
except KeyError:
if name_1 not in warned and name_1 not in name2index:
warned.add(name_1)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_1), cci))
if name_2 not in warned and name_2 not in name2index:
warned.add(name_2)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_2), cci))
if bonds:
bonds = np.array(bonds, int)
model.setBonds(bonds)
ideal.setBonds(bonds)
return dict_
def get_snap_status_by_volname(volname, mnode=None):
"""Parse the output of 'gluster snapshot status' command
for the given volume.
Args:
volname (str): snapshot name
Kwargs:
mnode (str): Node on which command has to be executed.
If None, defaults to nodes[0].
Returns:
NoneType: None if command execution fails, parse errors.
list: list of dicts on success.
Examples:
>>> get_snap_status_by_volname('testvol',
mnode = 'abc.lab.eng.xyz.com')
[{'volCount': '1', 'volume': {'brick': [{'path': '10.70.47.11:
testvol_brick0', 'pid': '26747', 'lvUsage': '3.52', 'volumeGroup':
'RHS_vg0', 'lvSize': '9.95g'}, {'path': '10.70.47.16:/testvol_brick1',
'pid': '25497', 'lvUsage': '3.52', 'volumeGroup': 'RHS_vg0',
'lvSize': '9.95g'}], 'brickCount': '2'}, 'name': 'snap2', 'uuid':
'56a39a92-c339-47cc-a8b2-9e54bb2a6324'}, {'volCount': '1', 'volume':
{'brick': [{'path': '10.70.47.11:testvol_next_brick0', 'pid': '26719',
'lvUsage': '4.93', 'volumeGroup': 'RHS_vg1', 'lvSize': '9.95g'}],
'brickCount': '1'}, 'name': 'next_snap1',
'uuid': 'dcf0cd31-c0db-47ad-92ec-f72af2d7b385'}]
"""
if mnode is None:
mnode = tc.servers[0]
cmd = "gluster snapshot status volume %s --xml" % volname
ret, out, _ = tc.run(mnode, cmd, verbose=False)
if ret != 0:
tc.logger.error("Failed to execute 'snapshot status' on node %s. "
"Hence failed to get the snapshot status.", mnode)
return None
try:
root = etree.XML(out)
except etree.ParseError:
tc.logger.error("Failed to parse the gluster snapshot "
"status xml output.")
return None
snap_status_list = []
for snap in root.findall("snapStatus/snapshots/snapshot"):
snap_status = {}
for element in snap.getchildren():
if element.tag == "volume":
status = {}
status["brick"] = []
for elmt in element.getchildren():
if elmt.tag == "brick":
brick_info = {}
for el in elmt.getchildren():
brick_info[el.tag] = el.text
status["brick"].append(brick_info)
else:
status[elmt.tag] = elmt.text
snap_status[element.tag] = status
else:
snap_status[element.tag] = element.text
snap_status_list.append(snap_status)
return snap_status_list
