def _extract_text(self, elem):
# note the whitespace on the join here.
return u' '.join([
s.decode("utf-8")
for s in ET.tostringlist(elem, method="text", encoding="utf-8")
if s is not None
]).strip() # don't ask...
def save(self, filename):
''' Write current contents of database to xml
'''
# TODO: Check for overwrite existing
#with sql.connect(self.dbfilename) as con:
# Get list of objects with objname
self.dbcon.row_factory = sql.Row
cur = self.dbcon.cursor()
cur.execute("SELECT name FROM sqlite_master WHERE type='table'")
tables = [t[0] for t in cur.fetchall()]
#with open(filename,'w',encoding='utf-8') as f:
with codecs.open(filename, "w", "utf-8-sig") as f:
# file writing in Python3 is different than 2, have to convert
# strings to bytes or open the file with an encoding. There is no
# easy write for all data types
#f.write('\ufeff') # In order to match existing file
#f.write(codecs.BOM_UTF8)
f.write(
'<MasterDataSet xmlns="http://tempuri.org/MasterDataSet.xsd">\r\n'
)
for table_name in sorted(tables):
#for d_type in order:
# table_name = d_type[2:]
try:
cur.execute("SELECT * FROM '%s'" % (table_name))
except:
#print("Bad table %s"%table_name)
continue
row_keys = [k[0] for k in cur.description]
#cElementTree has no pretty print, so some convolution is needed
row = cur.fetchone()
while row is not None:
f.write(' ')
ele = etree.Element('t_' + table_name)
for (sube, val) in zip(row_keys, row):
if val is None:
continue
attr_ele = etree.SubElement(ele, sube)
if isinstance(val, int):
val = str(val)
attr_ele.text = val
ele_slist = etree.tostringlist(ele)
''' This is done because in python2, to_string prepends the string with an
xml declaration. Also in python2, the base class of 'bytes' is basestring
TODO: Will this ever process an element with no data?
'''
if isinstance(ele_slist[0], str):
ele_s = "".join(ele_slist)
else:
# Python3 bytes object
ele_s = ""
for bl in ele_slist:
ele_s += bl.decode('UTF-8')
f.write(
ele_s.replace('><',
'>\r\n <').replace(' </t_', '</t_'))
f.write('\r\n')
row = cur.fetchone()
f.write('</MasterDataSet>\r\n')
def _ET2unicode(self, ET_instance):
"returns unicode of elementtree contents"
if ET_instance is not None:
return (" ".join(
ET.tostringlist(ET_instance, method="text",
encoding="utf-8"))).strip()
else:
return ""
def tree_to_unicode(self, tree):
"""
returns unicode of elementtree contents (in an unnecessarily complicated way!)
cochrane XML may contain HTML, this allows it to be extracted properly
"""
if tree is not None:
return (" ".join(ET.tostringlist(tree, method="text", encoding="utf-8"))).strip()
else:
return None
def test_zillow_api_results(self):
"""
"""
address = '2114 Bigelow Ave Seattle, WA'
zipcode = '98109'
zpid = '48749425'
# create expected result from mock data
deep_search_mock = ElementTree.XML(MOCK_DATA_GET_DEEP_SEARCH_RESULTS)
# create response from zillow
zillow_data = ZillowWrapper(ZILLOW_API_KEY)
deep_search_response = zillow_data.get_deep_search_results(address, zipcode)
# compare expected with real result
self.assertEqual(ElementTree.tostringlist(deep_search_response, encoding="us-ascii", method="xml"),
ElementTree.tostringlist(deep_search_mock, encoding="us-ascii", method="xml"))
def save(db, filename):
''' Write contents of plexos mongo database to xml filename
Args: db - mongo database
filename - Location to save plexos XML file. The file will be
overwritten if it exists
No Return
'''
# TODO: Check for overwrite existing xml
with codecs.open(filename, "w", "utf-8-sig") as fout:
# file writing in Python3 is different than 2, have to convert
# strings to bytes or open the file with an encoding. There is no
# easy write for all data types
# TODO: Support all root_element and namespace defs
all_orders = db[META_TABLE].find({}, {'_id': 0})
if all_orders.count() == 0:
raise Exception("No metadata availble to write file")
order = next(all_orders)
t_order = order['t_order']
e_order = order['e_order']
fout.write('<%s xmlns="%s">\r\n' %
("MasterDataSet", "http://tempuri.org/MasterDataSet.xsd"))
for col in t_order:
for doc in db[col].find({}, {'_id': 0}):
fout.write(' ')
ele = etree.Element('t_' + col)
for sube in e_order[col]:
if sube not in doc:
continue
val = doc[sube]
# Uncommenting the following will ignore subelements with no values
# Sometimes missing subelements with no values were crashing plexos.
# See issue #54
#if val is None:
# continue
attr_ele = etree.SubElement(ele, sube)
if isinstance(val, int) or isinstance(val, float):
val = str(val)
attr_ele.text = val
ele_slist = etree.tostringlist(ele)
# This is done because in python2, to_string prepends the string with an
# xml declaration. Also in python2, the base class of 'bytes' is basestring
# TODO: Will this ever process an element with no data?
if isinstance(ele_slist[0], str):
ele_s = "".join(ele_slist)
else:
# Python3 bytes object
ele_s = ""
for byte_list in ele_slist:
ele_s += byte_list.decode('UTF-8')
fout.write(
ele_s.replace('><',
'>\r\n <').replace(' </t_', '</t_'))
fout.write('\r\n')
fout.write('</%s>\r\n' % "MasterDataSet")
def sitemap(request):
response = HttpResponse(content_type='text/xml')
response['Content-Disposition'] = 'attachment; filename="sitemap.xml"'
root = ET.Element("root")
doc = ET.SubElement(root, "doc")
ET.SubElement(doc, "field1", name="blah").text = "some value1"
ET.SubElement(doc, "field2", name="asdfasd").text = "some vlaue2"
return render(request, ET.tostringlist(root), {}, content_type="text/xml")
def tree_to_unicode(self, tree):
"""
returns unicode of elementtree contents (in an unnecessarily complicated way!)
cochrane XML may contain HTML, this allows it to be extracted properly
"""
if tree is not None:
return (" ".join(
ET.tostringlist(tree, method="text",
encoding="utf-8"))).strip()
else:
return None
def saveSVG(self, filename, doc):
"""
........Save doc (ElementTreeDoc) content as a svg file.
........"""
f = open(filename, 'w')
f.write('<?xml version="1.0" standalone="no"?>\n')
f.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"\n')
f.write('"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n')
f.write('\n'.join(ET.tostringlist(doc)))
f.close()
return ()
def save(file_path, model):
"""
Save the model onto the file identified by file_path.
Parameters
----------
file_path : str
The path to the filename where the model has to be saved
model : RTEnsemble
The model RTEnsemble model to save on file
Returns
-------
status : bool
Returns true if the save is successful, false otherwise
"""
# ROOT
root = etree.Element("ranker")
# Learning process info
n_info = etree.SubElement(root, "info")
etree.SubElement(n_info, "type").text = "MART"
etree.SubElement(n_info, "trees").text = repr(model.n_trees)
etree.SubElement(n_info, "leaves").text = repr(model.max_leaves())
etree.SubElement(n_info, "shrinkage").text = repr(model.learning_rate)
etree.SubElement(n_info, "leafsupport").text = repr(0)
etree.SubElement(n_info, "discretization").text = repr(0)
etree.SubElement(n_info, "estop").text = repr(0)
# Ensemble
n_ensemble = etree.Element("ensemble")
root.append(n_ensemble)
for idx_tree in xrange(model.n_trees):
n_tree = etree.SubElement(
n_ensemble, "tree", {
"id": repr(idx_tree + 1),
"weight": repr(model.trees_weight[idx_tree])
})
cur_node = model.trees_root[idx_tree]
n_split = ProxyQuickRank._get_node_xml_element(model, cur_node)
n_tree.append(n_split)
with open(file_path, 'w') as f_out:
print >> f_out, ''.join(
ProxyQuickRank._xmlprettyprint(etree.tostringlist(root)))
return True
def ToElementString(self, element):
strList = ElementTree.tostringlist(element)
if element.tag == 'IterateVolumeElements':
outStr = element.attrib['XPath']
return outStr
outStr = ""
for s in strList:
outStr = outStr + " " + s.decode('Utf-8')
if s == '>':
break
return outStr
def _ET2unicode(self, ET_instance, strip_tags=True):
"returns unicode of elementtree contents"
if ET_instance is not None:
if strip_tags:
# print "tags stripped!"
return ' '.join([
s for s in ET.tostringlist(
ET_instance, method="text", encoding="unicode")
if s is not None
])
else:
return ET.tostring(ET_instance, method="xml",
encoding="utf-8").decode("utf-8")
else:
return u""
def maintainers_get(apiurl, project, package=None):
if package is None:
meta = ETL.fromstringlist(show_project_meta(apiurl, project))
maintainers = meta.xpath('//person[@role="maintainer"]/@userid')
groups = meta.xpath('//group[@role="maintainer"]/@groupid')
maintainers.extend(groups_members(apiurl, groups))
return maintainers
# Ugly reparse, but real xpath makes the rest much cleaner.
root = owner_fallback(apiurl, project, package)
root = ETL.fromstringlist(ET.tostringlist(root))
maintainers = root.xpath('//person[@role="maintainer"]/@name')
groups = root.xpath('//group[@role="maintainer"]/@name')
maintainers.extend(groups_members(apiurl, groups))
return maintainers
def maintainers_get(apiurl, project, package=None):
if package is None:
meta = ETL.fromstringlist(show_project_meta(apiurl, project))
maintainers = meta.xpath('//person[@role="maintainer"]/@userid')
groups = meta.xpath('//group[@role="maintainer"]/@groupid')
maintainers.extend(groups_members(apiurl, groups))
return maintainers
# Ugly reparse, but real xpath makes the rest much cleaner.
root = owner_fallback(apiurl, project, package)
root = ETL.fromstringlist(ET.tostringlist(root))
maintainers = root.xpath('//person[@role="maintainer"]/@name')
groups = root.xpath('//group[@role="maintainer"]/@name')
maintainers.extend(groups_members(apiurl, groups))
return maintainers
def _request(self, xml):
"""Given an ElementTree.Element, serialize it and do the request.
Returns an ElementTree.Element containing the response"""
if not self.protocol:
# Do a bogus request, using the HTTP protocol. If there is no
# header (see special case in communicate(), we should be using the
# raw protocol
header, data = self._communicate('<RIBCL VERSION="2.0"></RIBCL>',
ILO_HTTP)
if header:
self.protocol = ILO_HTTP
else:
self.protocol = ILO_RAW
# Serialize the XML
xml = "\r\n".join(etree.tostringlist(xml)) + '\r\n'
header, data = self._communicate(xml, self.protocol)
# This thing usually contains multiple XML messages
messages = []
while data:
pos = data.find('<?xml', 5)
if pos == -1:
message = self._parse_message(data)
data = None
else:
message = self._parse_message(data[:pos])
data = data[pos:]
# _parse_message returns None if a message has no useful content
if message is not None:
messages.append(message)
if not messages:
return header, None
elif len(messages) == 1:
return header, messages[0]
else:
return header, messages
def record_to_xml(record, quiet=False, namespace=False):
"""
converts a record object to a chunk of xml
# include the marcxml namespace in the root tag (default: False)
record_to_xml(record, namespace=True)
"""
# helper for converting non-unicode data to unicode
# TODO: maybe should set g0 and g1 appropriately using 066 $a and $b?
# marc8 = MARC8ToUnicode(quiet=quiet)
# def translate(data):
# if type(data) == unicode:
# return data
# else:
# return marc8.translate(data)
root = ET.Element('record')
if namespace:
root.set('xmlns', MARC_XML_NS)
root.set('xmlns:xsi', XSI_NS)
root.set('xsi:schemaLocation', MARC_XML_SCHEMA)
leader = ET.SubElement(root, 'leader')
leader.text = record.leader
for field in record:
if field.is_control_field():
control_field = ET.SubElement(root, 'controlfield')
control_field.set('tag', field.tag)
control_field.text = field.data
else:
data_field = ET.SubElement(root, 'datafield')
data_field.set('tag', field.tag)
data_field.set('ind1', field.indicators[0])
data_field.set('ind2', field.indicators[1])
for subfield in field:
data_subfield = ET.SubElement(data_field, 'subfield')
data_subfield.set('code', subfield[0])
data_subfield.text = subfield[1]
return replace_illegal(''.join(ET.tostringlist(root, encoding='utf-8')))
def _request(self, xml):
"""Given an ElementTree.Element, serialize it and do the request.
Returns an ElementTree.Element containing the response"""
if not self.protocol:
# Do a bogus request, using the HTTP protocol. If there is no
# header (see special case in communicate(), we should be using the
# raw protocol
header, data = self._communicate('<RIBCL VERSION="2.0"></RIBCL>', ILO_HTTP)
if header:
self.protocol = ILO_HTTP
else:
self.protocol = ILO_RAW
# Serialize the XML
xml = "\r\n".join(etree.tostringlist(xml)) + '\r\n'
header, data = self._communicate(xml, self.protocol)
# This thing usually contains multiple XML messages
messages = []
while data:
pos = data.find('<?xml', 5)
if pos == -1:
message = self._parse_message(data)
data = None
else:
message = self._parse_message(data[:pos])
data = data[pos:]
# _parse_message returns None if a message has no useful content
if message is not None:
messages.append(message)
if not messages:
return header, None
elif len(messages) == 1:
return header, messages[0]
else:
return header, messages
def xmlToTxt (self, pmcid):
noTxt = False
fileName = 'full text/pmcid/xml/'+ str(pmcid)
directory = os.path.dirname(fileName)
if not os.path.exists(directory):
os.makedirs(directory)
tree = ET.parse(fileName + '.xml')
root = tree.getroot()
rootNS, _ = root.tag.split('}')
rootNS = rootNS [1 :]
nsmap = {'rootNS':rootNS}
metadata = root.find('.//rootNS:metadata', nsmap)
try:
article = metadata[0]
except TypeError:
article = None
noTxt = True
if article:
articleNS, temp = article.tag.split('}')
assert temp == 'article', 'No article tag'
del(temp)
articleNS = articleNS[1 :]
nsmap['articleNS'] = articleNS
body = article.find('.//articleNS:body', nsmap)
if body:
txt = ' '.join(ET.tostringlist(body, encoding='unicode', method='text'))
fileName = 'full text/pmcid/txt/' +str(pmcid)+ '.txt'
directory = os.path.dirname(fileName)
if not os.path.exists(directory):
os.makedirs(directory)
with open('full text/pmcid/txt/' +str(pmcid)+ '.txt', 'wb') as txtFile:
txtFile.write(txt.encode('utf-8'))
else:
noTxt = True
return noTxt
def write_new_file(infile, outfile):
'''writes a new OSM file with fixed street names'''
with open(outfile, 'w') as outfile:
outfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
outfile.write('<osm>\n ')
counter = 0
for element in get_element(infile):
if element.tag == 'node':
if element.findall('tag'):
for node in element:
if node.attrib['k'] == 'addr:street':
new_name = update_name(node.attrib['v'],
better_st_name)
node.attrib['v'] = new_name
outfile.write(ET.tostring(element, encoding='utf-8'))
outfile.write('</node>\n ')
if element.findall('tag') == []:
outfile.write(ET.tostring(element, encoding='utf-8'))
if element.tag == 'way':
element_top = ET.tostringlist(element, encoding='utf-8')
outfile.write(''.join(map(str, element_top[0:8])))
outfile.write('\n ')
for node in element:
if node.tag == 'tag':
if node.attrib['k'] == 'addr:street':
new_name = update_name(node.attrib['v'],
better_st_name)
node.attrib['v'] = new_name
outfile.write(ET.tostring(node, encoding='utf-8'))
else:
outfile.write(ET.tostring(node, encoding='utf-8'))
else:
outfile.write(ET.tostring(node, encoding='utf-8'))
outfile.write('</way>\n ')
if element.tag == 'relation':
ET.tostring(element, encoding='utf-8')
outfile.write('</osm>')
def _request(self, xml, progress=None):
"""Given an ElementTree.Element, serialize it and do the request.
Returns an ElementTree.Element containing the response"""
if not self.protocol:
self._detect_protocol()
# Serialize the XML
if hasattr(etree, 'tostringlist'):
xml = "\r\n".join(etree.tostringlist(xml)) + '\r\n'
else:
xml = etree.tostring(xml)
header, data = self._communicate(xml, self.protocol, progress=progress)
# This thing usually contains multiple XML messages
messages = []
while data:
pos = data.find('<?xml', 5)
if pos == -1:
message = self._parse_message(data)
data = None
else:
message = self._parse_message(data[:pos])
data = data[pos:]
# _parse_message returns None if a message has no useful content
if message is not None:
messages.append(message)
if not messages:
return header, None
elif len(messages) == 1:
return header, messages[0]
else:
return header, messages
def _request(self, xml, progress=None):
"""Given an ElementTree.Element, serialize it and do the request.
Returns an ElementTree.Element containing the response"""
if not self.protocol:
self._detect_protocol()
# Serialize the XML
if hasattr(etree, 'tostringlist'):
xml = b("\r\n").join(etree.tostringlist(xml)) + b('\r\n')
else:
xml = etree.tostring(xml)
header, data = self._communicate(xml, self.protocol, progress=progress)
# This thing usually contains multiple XML messages
messages = []
while data:
pos = data.find('<?xml', 5)
if pos == -1:
message = self._parse_message(data)
data = None
else:
message = self._parse_message(data[:pos])
data = data[pos:]
# _parse_message returns None if a message has no useful content
if message is not None:
messages.append(message)
if not messages:
return header, None
elif len(messages) == 1:
return header, messages[0]
else:
return header, messages
def record_to_rustam_xml(record, syntax='1.2.840.10003.5.28', quiet=False, namespace=False):
"""
converts a record object to a chunk of xml
# include the marcxml namespace in the root tag (default: False)
record_to_xml(record, namespace=True)
"""
# helper for converting non-unicode data to unicode
# TODO: maybe should set g0 and g1 appropriately using 066 $a and $b?
# marc8 = MARC8ToUnicode(quiet=quiet)
# def translate(data):
# if type(data) == unicode:
# return data
# else:
# return marc8.translate(data)
root = ET.Element('record')
root.set('syntax', syntax)
# if namespace:
# root.set('xmlns', MARC_XML_NS)
# root.set('xmlns:xsi', XSI_NS)
# root.set('xsi:schemaLocation', MARC_XML_SCHEMA)
leader = ET.SubElement(root, 'leader')
#leader.text = record.leader
length = ET.SubElement(leader, 'length')
length.text = record.leader[0:5]
status = ET.SubElement(leader, 'status')
status.text = record.leader[5]
type = ET.SubElement(leader, 'type')
type.text = record.leader[6]
leader07 = ET.SubElement(leader, 'leader07')
leader07.text = record.leader[7]
leader08 = ET.SubElement(leader, 'leader08')
leader08.text = record.leader[8]
leader09 = ET.SubElement(leader, 'leader09')
leader09.text = record.leader[9]
indicatorCount = ET.SubElement(leader, 'indicatorCount')
indicatorCount.text = record.leader[10]
identifierLength = ET.SubElement(leader, 'identifierLength')
identifierLength.text = record.leader[11]
dataBaseAddress = ET.SubElement(leader, 'dataBaseAddress')
dataBaseAddress.text = record.leader[12:17]
leader17 = ET.SubElement(leader, 'leader17')
leader17.text = record.leader[17]
leader18 = ET.SubElement(leader, 'leader18')
leader18.text = record.leader[18]
leader19 = ET.SubElement(leader, 'leader19')
leader19.text = record.leader[19]
entryMap = ET.SubElement(leader, 'entryMap')
entryMap.text = record.leader[20:24]
for field in record:
if field.is_control_field():
control_field = ET.SubElement(root, 'field')
control_field.set('id', field.tag)
control_field.text = field.data
elif field.tag[0] == '4':
data_field = ET.SubElement(root, 'field')
data_field.set('id', field.tag)
ind1 = ET.SubElement(data_field, 'indicator')
ind1.set('id', '1')
ind1.text = field.indicators[0]
ind2 = ET.SubElement(data_field, 'indicator')
ind2.set('id', '2')
ind2.text = field.indicators[1]
for subfield in field:
data_subfield = ET.SubElement(data_field, 'subfield')
data_subfield.set('id', subfield[0])
#data_subfield.text = translate(subfield[1])
if isinstance(subfield[1], Field):
inner_field = subfield[1]
inner_data_field = ET.SubElement(data_subfield, 'field')
inner_data_field.set('id', inner_field.tag)
if int(inner_field.tag) < 10:
inner_data_field.text = inner_field.data
continue
inner_ind1 = ET.SubElement(inner_data_field, 'indicator')
inner_ind1.set('id', '1')
inner_ind1.text = inner_field.indicators[0]
inner_ind2 = ET.SubElement(inner_data_field, 'indicator')
inner_ind2.set('id', '2')
inner_ind2.text = inner_field.indicators[1]
for inner_subfield in inner_field:
inner_data_subfield = ET.SubElement(inner_data_field, 'subfield')
inner_data_subfield.set('id', inner_subfield[0])
inner_data_subfield.text = inner_subfield[1]
else:
data_field = ET.SubElement(root, 'field')
data_field.set('id', field.tag)
ind1 = ET.SubElement(data_field, 'indicator')
ind1.set('id', '1')
ind1.text = field.indicators[0]
ind2 = ET.SubElement(data_field, 'indicator')
ind2.set('id', '2')
ind2.text = field.indicators[1]
for subfield in field:
data_subfield = ET.SubElement(data_field, 'subfield')
data_subfield.set('id', subfield[0])
data_subfield.text = subfield[1]
return replace_illegal(''.join(ET.tostringlist(root, encoding='utf-8')))
nx_layouts = [
nx.circular_layout(g_nx, 10),
nx.shell_layout(g_nx, None, 7, None, 2),
nx.spring_layout(g_nx, None, None, None, 50, 1e-4, None, 7),
nx.kamada_kawai_layout(g_nx, None, None, 'weight', 10, None, 2),
nx.spectral_layout(g_nx, 'weight', 20, None, 2)
]
nx_lay = nx_layouts[n]
nx_doc = ET.parse(graph)
nx_root = nx_doc.getroot()
for nl in range(len(nx_lay)):
nx_node = nx_root.find('graph').findall('node')[nl].attrib
nx_node['x'] = str(round(nx_lay[nx_node['id']][0], 5))
nx_node['y'] = str(round(nx_lay[nx_node['id']][1], 5))
g.message("sending Message..")
g.message(ET.tostringlist(nx_doc.getroot(), encoding="utf8", method="xml"))
g.message("..Message send")
#g.message("sending Graph..")
#g.setGraph("xml",ET.tostringlist(nx_doc.getroot(), encoding="utf8", method="xml"))
#g.message("..Graph send")
ig_layouts = [
"circle", "star", "grid", "graphopt", "fruchterman_reingold",
"kamada_kawai", "mds", "lgl", "reingold_tilford",
"reingold_tilford_circular"
] #,"sugiyama"]
ig_lay = g_ig.layout(ig_layouts[n])
ig_lay.scale(len(ig_layouts) / 2)
ig_doc = ET.parse(graph)
ig_root = ig_doc.getroot()
for il in range(len(ig_lay)):
def xmlprettyprint(element):
return ''.join(_xmlprettyprint(ET.tostringlist(element,'utf-8')))
def convert_bdt__Grad(bdt_clf, input_var_list, tmva_outfile_xml):
NTrees = bdt_clf.n_estimators
learning_rate = bdt_clf.learning_rate
var_list = input_var_list
MethodSetup = ET.Element("MethodSetup", Method="BDT::BDT")
GeneralInfo = ET.SubElement(MethodSetup, "GeneralInfo")
Info_Creator = ET.SubElement(
GeneralInfo, "Info", name="Creator", value="VBF-learn (Yacine Haddad)"
)
Info_AnalysisType = ET.SubElement(
GeneralInfo, "Info", name="AnalysisType", value="Classification"
)
# <Options>
Options = ET.SubElement(MethodSetup, "Options")
Option_NodePurityLimit = ET.SubElement(
Options, "Option", name="NodePurityLimit", modified="No"
).text = str(NodePurityLimit)
Option_BoostType = ET.SubElement(
Options, "Option", name="BoostType", modified="Yes"
).text = "Grad"
Option_NTrees = ET.SubElement(
Options, "Option", name='NTrees', modified="Yes"
).text = str(bdt_clf.n_estimators)
Option_MaxDepth = ET.SubElement(
Options, "Option", name='MaxDepth', modified="Yes"
).text = str(bdt_clf.max_depth)
Option_Shrinkage = ET.SubElement(
Options, "Option", name="Shrinkage", modified="Yes"
).text = str(bdt_clf.learning_rate)
Option_UseNvars = ET.SubElement(
Options, "Option", name='UseNvars', modified="Yes"
).text = str(bdt_clf.max_features)
# <Variables>
Variables = ET.SubElement(MethodSetup, "Variables", NVar=str(len(var_list)))
for ind, val in enumerate(var_list):
max_val = str(X_train[:,ind].max())
min_val = str(X_train[:,ind].min())
print(min_val, max_val)
Variable = ET.SubElement(
Variables, "Variable", VarIndex=str(ind), Type='F',
Expression=val, Label=val, Title=val, Unit="", Internal=val,
Min=min_val, Max=max_val
)
# <Spectators>
Spectators = ET.SubElement(MethodSetup, 'Spectators', NSpec='0')
# <Classes>
Classes = ET.SubElement(MethodSetup, "Classes", NClass='2')
class_Creator = ET.SubElement(Classes, 'class', Name='Signal', Index='0')
class_Creator = ET.SubElement(Classes, 'class', Name='Background', Index='1')
# <Transformations>
Transformations = ET.SubElement(
MethodSetup, 'Transformations', NTransformations='0'
)
# <MVAPdfs>
MVAPdfs = ET.SubElement(MethodSetup, 'MVAPdfs')
# <Weights>
Weights = ET.SubElement(MethodSetup, "Weights", NTrees=str(NTrees), AnalysisType="1")
for idx, dt in enumerate(bdt_clf.estimators_[:, 0]):
BinaryTree = ET.SubElement(Weights, "BinaryTree", type="DecisionTree", boostWeight="1.0e+00", itree=str(idx))
build_xml_tree__Grad(dt, 0, "s", -1, BinaryTree,learning_rate)
XMLtext= ET.tostringlist(MethodSetup)
OutputStr=''
level=-1
for t in XMLtext:
if t[0]=='<':
level=level+1
if t[0:2]=='</':
#remove the added level because of '<'
level=level-1
if t[-1]=='"' or t[0]=='<' or t[-2:]=='/>':
t="\n"+" "*level+t
if t[-2:]=='/>' or t[level*4+1:level*4+3]=='</': #end of block
level=level-1
#if t[-1:]=='>':
# t=t+'\n'
OutputStr=OutputStr+t
f = open(tmva_outfile_xml, "w")
f.write(OutputStr)
f.close()
def _ET2unicode(self, ET_instance):
"returns unicode of elementtree contents"
if ET_instance is not None:
return (" ".join(ET.tostringlist(ET_instance, method="text", encoding="utf-8"))).strip()
else:
return ""
def drawSVG(self):
'''
Draw SVG based on the orf annotation
'''
x = 50
y = 70
rightMargin = 100
fontSize = 16
boxHeight = 20
maxLength = max(self.lengthDic)
maxTitleLength = len(max(self.organismDic, key=lambda x:len(x)))
self.setStandard()
extraLeftMargin =0
if min(self.xDeltaDic)<0:
extraLeftMargin =abs(min(self.xDeltaDic))
else:
extraLeftMargin = -abs(min(self.xDeltaDic))
extraRightMargin = max(self.xDeltaDic)
canvasWidth = int((maxLength+extraLeftMargin+extraRightMargin) * self.scaleFactor)
leftMargin = int(maxTitleLength * fontSize * 0.65)+int(extraLeftMargin*self.scaleFactor)+50
effectiveWidth = canvasWidth - leftMargin - rightMargin
conversion = float(effectiveWidth) / float(maxLength)
yDelta = 60
canvasHeight = len(self.results) * yDelta + 100
# doc is elementTree container for svg
doc = ET.Element('svg', width=str(canvasWidth),
height=str(canvasHeight), version='1.2',
xmlns='http://www.w3.org/2000/svg')
doc.attrib['xmlns:xlink'] = 'http://www.w3.org/1999/xlink'
# Color and Gradient Assignment
defs = self.colorDef()
doc.append(defs)
# Draw several domain arcorfecture in single SVG
for i in range(len(self.results)):
(organismName, organism, start, end, length, specie) = self.results[i]
doc = self.singleLabel(
i,
specie,
doc,
x,
y,
leftMargin,
fontSize,
conversion,
boxHeight,
)
doc = self.singleSVG(
i,
specie,
doc,
x,
y,
leftMargin,
fontSize,
conversion,
boxHeight,
)
y += yDelta
self.saveSVG(self.outputSVG, doc)
return '\n'.join(ET.tostringlist(doc))
def drawSVG(self):
# Draw SVG based on the hmmer domaim
x = 50
y = 70
leftMargin = 200
rightMargin = 100
fontSize = 16
boxHeight = 20
(maxLength, maxTitleLength) = self.maxLength()
canvasWidth = int(maxLength * self.scaleFactor)
effectiveWidth = canvasWidth - leftMargin - rightMargin
conversion = float(effectiveWidth) / float(maxLength)
# If protein name width is bigger than leftMargin, wrote Label at top of proteins (self.titlemode=True)
if maxTitleLength * fontSize * conversion * 0.7 > leftMargin:
self.titlemode = True
else:
self.titlemode = False
if self.titlemode:
yDelta = 110 + boxHeight * len(self.hmmerResults[0].tier)
leftMargin = 50
effectiveWidth = canvasWidth - leftMargin - rightMargin
conversion = float(effectiveWidth) / float(maxLength)
else:
yDelta = 80
canvasHeight = len(self.hmmerResults) * yDelta + 100
# doc is elementTree container for svg
doc = ET.Element('svg', width=str(canvasWidth),
height=str(canvasHeight), version='1.2',
xmlns='http://www.w3.org/2000/svg')
doc.attrib['xmlns:xlink'] = 'http://www.w3.org/1999/xlink'
# Color and Gradient Assignment
defs = self.colorDef()
doc.append(defs)
# Draw several domain architecture in single SVG
for hmmer in self.hmmerResults:
doc = self.singleSVG(
hmmer,
doc,
x,
y,
leftMargin,
fontSize,
conversion,
boxHeight,
)
y += yDelta
self.saveSVG(self.outputSVG, doc)
return '\n'.join(ET.tostringlist(doc))
def drawClusterTable(self,clusters,clusterNames,clusterColors):
'''
Draw SVG based on the orf annotation
'''
# Find Maximum Length of ORF (as nucleotide. Not amino acid!) in each Clusters
clusterLengthMax = {}
for (organismName, organism, start, end, length, specie) in self.results:
for orf in specie:
orfLength = orf.end - orf.start
if not orf.accession in clusterLengthMax:
clusterLengthMax[orf.accession]=orfLength
elif orfLength>clusterLengthMax[orf.accession]:
clusterLengthMax[orf.accession]=orfLength
y = 70
rightMargin = 100
fontSize = 16
boxHeight = 20
boxWidth = 20
# maxTitleLength = len(max(self.organismDic, key=lambda x:len(x)))
maxTitleLength = 0
leftMargin = int(maxTitleLength * fontSize * 0.65)
canvasWidth = int(leftMargin+len(clusters)*boxWidth+rightMargin)
# effectiveWidth = canvasWidth - leftMargin - rightMargin
yDelta = 25
canvasHeight = len(self.results) * yDelta + 100
# doc is elementTree container for svg
doc = ET.Element('svg', width=str(canvasWidth),
height=str(canvasHeight), version='1.2',
xmlns='http://www.w3.org/2000/svg')
doc.attrib['xmlns:xlink'] = 'http://www.w3.org/1999/xlink'
# Color and Gradient Assignment
defs = self.colorDef()
doc.append(defs)
clusterlink = self.linkAddress+'#{0}'
# Draw Top Cluster Labels
for i in range(len(clusters)):
x = leftMargin + i* boxWidth
clusterColor = clusterColors[i]
clusterName = clusterNames[i]
link = ET.Element('a')
link.attrib['xlink:href'] = clusterlink.format(clusterName.replace(" ", ""))
label = ET.Element('text', x=str(x),
y=str(y-20),
fill=clusterColor,
style='font-family:Sans-Serif;font-weight:bold;font-size:16px;text-anchor:left;dominant-baseline:middle'
)
label.text = str(i+1)
link.append(label)
doc.append(link)
# Draw several domain architure in single SVG
for j in range(len(self.results)):
(organismName, organism, start, end, length, specie) = self.results[j]
for i in range(len(clusters)):
x = leftMargin + i* boxWidth
clusterColor = clusterColors[i]
clusterName = clusterNames[i]
for orf in specie:
if orf.accession == clusterName:
ratio = float(orf.end-orf.start)/float(clusterLengthMax[orf.accession])
# if ratio>1:
# print clusterName, ratio, orf.end-orf.start, clusterLengthMax[orf.accession], orf.accession
if orf.border:
if orf.accession ==self.standardCluster:
border = ';stroke-width:3;stroke:red'
else:
border = ';stroke-width:1;stroke:black'
else:
border = ''
style = border
link=ET.Element('a')
link.attrib['xlink:href']=orf.link
box = ET.Element('rect', x=str(x),
y=str(y),
width = str(int(boxWidth*ratio)),
height = str(boxHeight),
fill=clusterColor,
style=style
)
link.append(box)
doc.append(link)
y += yDelta
self.saveSVG(self.path+self.clusterTable, doc)
return '\n'.join(ET.tostringlist(doc))
