def loadLanguageFile(langfile):
""" Loads one or more languages from disk
Returns list of language codes found."""
global languagenames, languages
tree = ElementTree(file=langfile)
foundlangs = []
langslist = tree.find("wurmlanguages")
langs = langslist.findall("languagedef")
for language in langs:
code = language.attrib["code"]
name = language.attrib.get("name", code)
languagenames[code] = name
languages.setdefault(code, {}) # make sure the base map is ready
if code not in foundlangs:
foundlangs.append(code)
lstrings = language.findall("string")
for lstr in lstrings:
orig = _parse(lstr.find("original").text.strip())
tran = _parse(lstr.find("translation").text.strip())
languages[code][orig] = tran
return foundlangs
def fulltext_search(self, query, rows=None, start=None):
"""Does an advanced search on fulltext:blah.
You get back a pair (x,y) where x is the total # of hits
and y is a list of identifiers like ["foo", "bar", etc.]"""
query = self._prefix_query('fulltext', query)
result_list = self.raw_search(query, rows=rows, start=start)
e = ElementTree()
try:
e.parse(StringIO(result_list))
except SyntaxError as e:
raise SolrError(e)
total_nbr_text = e.find('info/range_info/total_nbr').text
# total_nbr_text = e.find('result').get('numFound') # for raw xml
total_nbr = int(total_nbr_text) if total_nbr_text else 0
out = []
for r in e.getiterator('hit'):
for d in r.find('metadata'):
for x in list(d.getiterator()):
if x.tag == "identifier":
xid = six.text_type(x.text).encode('utf-8')
if xid.startswith('OCA/'):
xid = xid[4:]
elif xid.endswith('.txt'):
xid = xid.split('/')[-1].split('_')[0]
elif xid.endswith('_ZZ'):
xid = xid[:-3]
out.append(xid)
break
return (total_nbr, out)
def load(cls, filename):
project_root = ElementTree().parse(filename)
settings_node = project_root.find(cls.__PROJECT_SETTINGS_NODE_NAME)
tracks_node = project_root.find(cls.__PROJECT_TRACKS_NODE_NAME)
track_nodes = tracks_node.findall(cls.__PROJECT_TRACK_NODE_NAME)
for node in track_nodes:
print node
def _get_article_contrib_authors(tree):
from sys import stderr
"""
Given an ElementTree, returns article authors in a format suitable for citation.
"""
authors = []
front = ElementTree(tree).find('front')
for contrib in front.iter('contrib'):
contribTree = ElementTree(contrib)
try:
surname = contribTree.find('name/surname').text
except AttributeError: # author is not a natural person
try:
citation_name = contribTree.find('collab').text
if citation_name is not None:
authors.append(citation_name)
continue
except AttributeError: # name has no immediate text node
continue
try:
given_names = contribTree.find('name/given-names').text
citation_name = ' '.join([surname, given_names[0]])
except AttributeError: # no given names
citation_name = surname
except TypeError: # also no given names
citation_name = surname
if citation_name is not None:
authors.append(citation_name)
return ', '.join(authors)
def _get_supplementary_material(tree, rid):
"""
Given an ElementTree and an rid, returns supplementary material as a dictionary
containing url, mimetype and label and caption.
"""
for sup in tree.iter('supplementary-material'):
try:
if sup.attrib['id'] == rid: # supplementary material found
result = {}
sup_tree = ElementTree(sup)
label = sup_tree.find('label')
result['label'] = ''
if label is not None:
result['label'] = label.text
caption = sup_tree.find('caption')
result['caption'] = ''
if caption is not None:
result['caption'] = ' '.join(caption.itertext())
media = sup_tree.find('media')
if media is not None:
result['mimetype'] = media.attrib['mimetype']
result['mime-subtype'] = media.attrib['mime-subtype']
result['url'] = _get_supplementary_material_url(
_get_pmcid(tree),
media.attrib['{http://www.w3.org/1999/xlink}href']
)
return result
except KeyError: # supplementary material has no ID
continue
def main():
super_concat = False
options, args = interface()
xml = ElementTree().parse(options.input,
parser=XMLParser(target=MyTreeBuilder()))
# delete the older subs. models from the xml file
for node in [
'HKYModel', 'gtrModel', 'siteModel', 'patterns', 'treeLikelihood'
]:
xml = delete_node(xml, node, 1)
# delete the kappa and frequency parameters in 'operators'
delete = [
'kappa', 'frequencies', 'alpha', 'pInv', 'ac', 'ag', 'at', 'cg', 'gt'
]
xml = delete_children_from_node(xml, xml.find('operators'), delete)
xml = delete_children_from_node(
xml,
xml.find('mcmc').find('posterior').find('prior'), delete)
# there are 2 log tags, disambiguated w/ id params. delete elements from
# the one we want (fileLog)
xml = delete_children_from_node(xml, get_file_log_section(xml), delete)
# jettison some comments
xml = comment_remover(xml, [
'The general time reversible', 'HKY substitution model', 'site model',
'The unique patterns from 1 to end', 'npatterns='
])
# load our substitution model information
substitution_models = cPickle.load(open(options.subs))
snippets = ElementTree().parse(options.params,
parser=XMLParser(target=MyTreeBuilder()))
# insert patterns on a per locus basis
insert_position = get_position(xml, 'alignment')
xml = insert_patterns_for_locus(xml, substitution_models, insert_position,
snippets)
# insert substitution models on a per locus basis
insert_position = get_position(xml, 'strictClockBranchRates')
xml, insert_position = insert_models_for_locus(xml, substitution_models,
insert_position, snippets)
# insert site models on a per locus basis
xml, insert_position = insert_site_models_for_locus(
xml, substitution_models, insert_position, snippets)
# insert tree likelihoods on a per locus basis
xml, insert_position = insert_tree_likelihoods_for_locus(
xml, substitution_models, insert_position, snippets)
# insert operators
xml = insert_operators_for_locus(xml, substitution_models, snippets)
write(xml, options.output)
pdb.set_trace()
"""model_names, site_names = get_xml_model_names(set(sub_models_from_modeltest.values()))
def __init__(self,
protocol = None,
host_name = None,
user_name = None,
password = None,
database_configuration = None,
test = False,
database_server_configuration_file_path = None,
sqlite_db_path = None):
if database_server_configuration_file_path is None:
database_server_configuration_file_path = os.path.join(os.environ['OPUS_HOME'], 'settings', 'database_server_configurations.xml')
if (protocol is None or test) and host_name is None and user_name is None and password is None:
if not os.path.exists(database_server_configuration_file_path):
raise Exception('You do not have a file %s storing information about your database server configurations. Cannot load database.'%database_server_configuration_file_path)
if database_configuration is None:
db_node = self._database_configuration_node()
else:
db_node = database_configuration
database_configuration = ElementTree(file = database_server_configuration_file_path).getroot().find(db_node)
if database_configuration is None:
raise Exception('Could not find an entry in %s for %s. Cannot load database.'%(database_server_configuration_file_path, db_node))
self.protocol = database_configuration.find('protocol').text
self.host_name = database_configuration.find('host_name').text
self.user_name = database_configuration.find('user_name').text
self.password = database_configuration.find('password').text
else:
if protocol is None:
self.protocol = get_default_database_engine()
else:
self.protocol = protocol.lower()
if host_name is None:
self.host_name = 'localhost'
else:
self.host_name = host_name
if user_name is None:
self.user_name = ''
else:
self.user_name = user_name
if password is None:
self.password = ''
else:
self.password = password
# If the password is the empty string or None, check if it is defined in the environment variable
# SQLPASSWORD - if so, use that.
if (self.password is None or self.password=='') and 'SQLPASSWORD' in os.environ:
self.password = os.environ['SQLPASSWORD']
self.sqlite_db_path = sqlite_db_path
def _getresponse(self):
r = self.h.getresponse()
if r.status == 200:
et = ElementTree()
et.parse(r)
if et.find('status').text == 'OK':
return et
raise NessusException(et.find('contents').text)
elif r.status == 403:
self.token = None
raise NessusSessionException('Session timed out.')
raise NessusException(r.read())
def _getresponse(self):
r = self.h.getresponse()
if r.status == 200:
et = ElementTree()
et.parse(r)
if et.find('status').text == 'OK':
return et
raise NessusException(et.find('contents').text)
elif r.status == 403:
self.token = None
raise NessusSessionException('Session timed out.')
raise NessusException(r.read())
def main():
super_concat = False
options, args = interface()
xml = ElementTree().parse(options.input, parser=XMLParser(target=MyTreeBuilder()))
# delete the older subs. models from the xml file
for node in ["HKYModel", "gtrModel", "siteModel", "patterns", "treeLikelihood"]:
xml = delete_node(xml, node, 1)
# delete the kappa and frequency parameters in 'operators'
delete = ["kappa", "frequencies", "alpha", "pInv", "ac", "ag", "at", "cg", "gt"]
xml = delete_children_from_node(xml, xml.find("operators"), delete)
xml = delete_children_from_node(xml, xml.find("mcmc").find("posterior").find("prior"), delete)
# there are 2 log tags, disambiguated w/ id params. delete elements from
# the one we want (fileLog)
xml = delete_children_from_node(xml, get_file_log_section(xml), delete)
# jettison some comments
xml = comment_remover(
xml,
[
"The general time reversible",
"HKY substitution model",
"site model",
"The unique patterns from 1 to end",
"npatterns=",
],
)
# load our substitution model information
substitution_models = cPickle.load(open(options.subs))
snippets = ElementTree().parse(options.params, parser=XMLParser(target=MyTreeBuilder()))
# insert patterns on a per locus basis
insert_position = get_position(xml, "alignment")
xml = insert_patterns_for_locus(xml, substitution_models, insert_position, snippets)
# insert substitution models on a per locus basis
insert_position = get_position(xml, "strictClockBranchRates")
xml, insert_position = insert_models_for_locus(xml, substitution_models, insert_position, snippets)
# insert site models on a per locus basis
xml, insert_position = insert_site_models_for_locus(xml, substitution_models, insert_position, snippets)
# insert tree likelihoods on a per locus basis
xml, insert_position = insert_tree_likelihoods_for_locus(xml, substitution_models, insert_position, snippets)
# insert operators
xml = insert_operators_for_locus(xml, substitution_models, snippets)
write(xml, options.output)
pdb.set_trace()
"""model_names, site_names = get_xml_model_names(set(sub_models_from_modeltest.values()))
def load(self):
"Nacteni ze souboru."
root = ElementTree(file=self.filename)
ret = self.Result()
size = int(root.find('world/cells').text)
ret.world = [[None for j in range(size)] for i in range(size)]
ret.species = int(root.find('world/species').text)
ret.iterations = int(root.find('world/iterations').text)
for organism in root.find('organisms'):
ret.world[int(organism.find('x_pos').text)][int(organism.find('y_pos').text)] = organism.find('species').text
return ret
def load(self):
"Nacteni ze souboru."
root = ElementTree(file=self.filename)
ret = self.Result()
size = int(root.find('world/cells').text)
ret.world = [[None for j in range(size)] for i in range(size)]
ret.species = int(root.find('world/species').text)
ret.iterations = int(root.find('world/iterations').text)
for organism in root.find('organisms'):
ret.world[int(organism.find('x_pos').text)][int(
organism.find('y_pos').text)] = organism.find('species').text
return ret
def extract_pos(link):
root_namespace_reg = re.compile("\{.*\}", re.IGNORECASE)
root_namespace = None
try:
data = urllib.urlopen(link)
tree = ElementTree()
parser = XMLParser(encoding="iso-8859-1")
tree.parse(data, parser=parser)
root_namespace = root_namespace_reg.findall(tree.getroot().tag)[0]
name = tree.find(".//{0}Document//{0}name".format(root_namespace)).text
# parse important datas
unparsed = tree.find(".//{0}Document//{0}Placemark//{0}description".format(root_namespace)).text
except Exception, e:
return
def localiseFontsConf (self, pathFonts, sysFontFolder) :
'''Sets the <dir> and <cachdir> to be the directory in which
the fonts.conf file exists. This helps to provide better
seperation of our fonts from the host system.'''
fileName = pathFonts + '/fonts.conf'
scrName = sysFontFolder + '/fonts.conf'
# First lets check to see if the fonts.conf file exists
if os.path.isfile(fileName) == False :
shutil.copy(scrName, fileName)
# Import this module for this process only (May need to move it
# if other processes ever need it)
from xml.etree.cElementTree import ElementTree
et = ElementTree(file = fileName)
path = os.path.dirname(os.path.abspath(fileName))
for p in ('dir', 'cachedir') :
et.find(p).text = path
# Write out the new font.conf file
if et.write(fileName, encoding = 'utf-8') == None :
self._log_manager.log('INFO', 'Fonts have been localised', 'true')
return
def search(query):
query = query.lower()
parsed = expr.parseString(query)[0]
res = parsed()
print
print("Searching for \"{}\"".format(query))
if res[1]:
print("Found {} results".format(len(res[0])))
for file_id in list(res[0])[:5]:
print(files[file_id])
words = parsed.words()
tree = ElementTree()
tree.parse(files[file_id])
body = tree.find('text').find('body')
for p in body:
words_copy = list(words)
at_least_one = False
for word in words_copy:
if word in word_reg_exp.findall(p.text.lower().encode('utf-8')):
at_least_one = True
words.remove(word)
if at_least_one:
print("----| {}".format(p.text.encode('utf-8')))
print
else:
print("Negative requests are not allowed")
print
def extractElement(self, elementpath, respdata):
try:
tree = ElementTree()
tree.parse(StringIO(respdata))
except:
return None
# Strip off the top-level item
if elementpath[0] == '/':
elementpath = elementpath[1:]
splits = elementpath.split('/', 1)
root = splits[0]
if tree.getroot().tag != root:
return None
elif len(splits) == 1:
return tree.getroot().text
else:
elementpath = splits[1]
e = tree.find(elementpath)
if e is not None:
return e.text
else:
return None
def get_hash_credential(self, xml_file):
tree = ElementTree(file=xml_file)
encrypted_hash = tree.find('Lib/Account/Credentials3')
if encrypted_hash is not None:
return encrypted_hash.text
else:
return False
def _getBugInfo(self, bugno):
def _formatEmail(e):
if e.get('name'):
return '%s <%s>' % (self._encode(e.get('name')), e.text)
return e.text
bugXML = utils.web.getUrlFd(self.bugzillaRoot +
'show_bug.cgi?ctype=xml&excludefield=attachmentdata&'
'excludefield=long_desc&excludefield=attachment&id=' +
str(bugno))
etree = ElementTree(file=bugXML)
bugRoot = etree.find('bug')
buginfo = {
'bug_id': bugRoot.find('bug_id').text,
'summary': self._encode(bugRoot.find('short_desc').text),
'creation_time': bugRoot.find('creation_ts').text,
'last_change_time': bugRoot.find('delta_ts').text,
'bug_severity': bugRoot.find('bug_severity').text,
'bug_status': bugRoot.find('bug_status').text,
'resolution': ' ' + bugRoot.find('resolution').text
if bugRoot.find('resolution') != None
else '',
'product': self._encode(bugRoot.find('product').text),
'component': bugRoot.find('component').text,
'reporter': _formatEmail(bugRoot.find('reporter')),
'assigned_to': _formatEmail(bugRoot.find('assigned_to')),
'url': self.bugzillaRoot + str(bugno),
}
bugXML.close()
return buginfo
def extractElement(self, elementpath, respdata):
try:
tree = ElementTree()
tree.parse(StringIO(respdata))
except:
return None
# Strip off the top-level item
if elementpath[0] == '/':
elementpath = elementpath[1:]
splits = elementpath.split('/', 1)
root = splits[0]
if tree.getroot().tag != root:
return None
elif len(splits) == 1:
return tree.getroot().text
else:
elementpath = splits[1]
e = tree.find(elementpath)
if e is not None:
return e.text
else:
return None
def fulltext_search(self, query, rows=None, start=None):
"""Does an advanced search on fulltext:blah.
You get back a pair (x,y) where x is the total # of hits
and y is a list of identifiers like ["foo", "bar", etc.]"""
query = self._prefix_query('fulltext', query)
result_list = self.raw_search(query, rows=rows, start=start)
e = ElementTree()
try:
e.parse(StringIO(result_list))
except SyntaxError as e:
raise SolrError(e)
total_nbr_text = e.find('info/range_info/total_nbr').text
# total_nbr_text = e.find('result').get('numFound') # for raw xml
total_nbr = int(total_nbr_text) if total_nbr_text else 0
out = []
for r in e.getiterator('hit'):
for d in r.find('metadata'):
for x in list(d.getiterator()):
if x.tag == "identifier":
xid = six.text_type(x.text).encode('utf-8')
if xid.startswith('OCA/'):
xid = xid[4:]
elif xid.endswith('.txt'):
xid = xid.split('/')[-1].split('_')[0]
elif xid.endswith('_ZZ'):
xid = xid[:-3]
out.append(xid)
break
return (total_nbr, out)
def read_macro(filename):
# Extract the OAD script code from ZEN macro file: *.czmac
print('Reading OAD Macro: ', filename)
tree = ElementTree(file=filename)
pyscript = tree.find('Text').text
return pyscript
def set_game_info(self, info):
self.info = info
tree = ElementTree()
tree.parse(info["path"])
link_node = tree.find("link")
if link_node is not None:
self.link_field.set_text(link_node.text.strip())
else:
self.link_field.set_text("")
def get_username(self, path):
xml_file = os.path.join(path, 'shared.xml')
if os.path.exists(xml_file):
tree = ElementTree(file=xml_file)
username = tree.find('Lib/Account/Default')
try:
return username.text
except Exception:
pass
def set_game_info(self, info):
self.info = info
tree = ElementTree()
tree.parse(info["path"])
link_node = tree.find("link")
if link_node is not None:
self.link_field.set_text(link_node.text.strip())
else:
self.link_field.set_text("")
def get_categories_from_pmid(pmid):
"""
Gets MeSH headings, returns those not deemed too broad.
"""
if not type(pmid) == int:
raise TypeError, "Cannot get Categories for PMID %s of type %s." % (pmid, type(pmid))
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml' % pmid
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
categories = []
for heading in tree.iterfind('PubmedArticle/MedlineCitation/MeshHeadingList/MeshHeading'):
htree = ElementTree(heading)
descriptor_text = htree.find('DescriptorName').text
if (htree.find('QualifierName') is not None) or \
(' ' in descriptor_text and not 'and' in descriptor_text):
categories.append(descriptor_text)
return categories
def get_username(self, path):
xml_file = os.path.join(path, u'shared.xml')
if os.path.exists(xml_file):
tree = ElementTree(file=xml_file)
username = tree.find('Lib/Account/Default')
try:
return win.string_to_unicode(username.text)
except Exception:
pass
return False
def _get_article_date(tree):
"""
Given an ElementTree, returns article date.
"""
article_meta = ElementTree(tree).find('front/article-meta')
for pub_date in article_meta.iter('pub-date'):
if pub_date.attrib['pub-type'] == 'epub':
pub_date_tree = ElementTree(pub_date)
year = int(pub_date_tree.find('year').text)
try:
month = int(pub_date_tree.find('month').text)
except AttributeError:
month = 1 # TODO: is this correct?
try:
day = int(pub_date_tree.find('day').text)
except AttributeError:
day = 1 # TODO: is this correct?
return str(date(year, month, day))
return ''
def list_by_path(manifest_name, path, cache):
"""
List ROS stacks or packages within the specified path.
The cache will be updated with the resource->path
mappings. list_by_path() does NOT returned cached results
-- it only updates the cache.
:param manifest_name: MANIFEST_FILE or STACK_FILE, ``str``
:param path: path to list resources in, ``str``
:param cache: path cache to update. Maps resource name to directory path, ``{str: str}``
:returns: complete list of resources in ROS environment, ``[str]``
"""
resources = []
path = os.path.abspath(path)
basename = os.path.basename
for d, dirs, files in os.walk(path, topdown=True, followlinks=True):
if 'CATKIN_IGNORE' in files:
del dirs[:]
continue # leaf
if PACKAGE_FILE in files:
# parse package.xml and decide if it matches the search criteria
root = ElementTree(None, os.path.join(d, PACKAGE_FILE))
is_metapackage = root.find('./export/metapackage') is not None
if (
(manifest_name == STACK_FILE and is_metapackage) or
(manifest_name == MANIFEST_FILE and not is_metapackage) or
manifest_name == PACKAGE_FILE
):
resource_name = root.findtext('name').strip(' \n\r\t')
if resource_name not in resources:
resources.append(resource_name)
if cache is not None:
cache[resource_name] = d
del dirs[:]
continue # leaf
if manifest_name in files:
resource_name = basename(d)
if resource_name not in resources:
resources.append(resource_name)
if cache is not None:
cache[resource_name] = d
del dirs[:]
continue # leaf
elif MANIFEST_FILE in files or PACKAGE_FILE in files:
# noop if manifest_name==MANIFEST_FILE, but a good
# optimization for stacks.
del dirs[:]
continue # leaf
elif 'rospack_nosubdirs' in files:
del dirs[:]
continue # leaf
# remove hidden dirs (esp. .svn/.git)
[dirs.remove(di) for di in dirs if di[0] == '.']
return resources
def get_pmid_from_doi(doi):
if not type(doi) == unicode:
raise TypeError, "Cannot get PMID for DOI %s of type %s." % (doi, type(doi))
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=%s' % doi
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
try:
return int(tree.find('IdList/Id').text)
except AttributeError:
return None
def list_by_path(manifest_name, path, cache):
"""
List ROS stacks or packages within the specified path.
The cache will be updated with the resource->path
mappings. list_by_path() does NOT returned cached results
-- it only updates the cache.
:param manifest_name: MANIFEST_FILE or STACK_FILE, ``str``
:param path: path to list resources in, ``str``
:param cache: path cache to update. Maps resource name to directory path, ``{str: str}``
:returns: complete list of resources in ROS environment, ``[str]``
"""
resources = []
path = os.path.abspath(path)
basename = os.path.basename
for d, dirs, files in os.walk(path, topdown=True, followlinks=True):
if 'CATKIN_IGNORE' in files:
del dirs[:]
continue # leaf
if PACKAGE_FILE in files:
# parse package.xml and decide if it matches the search criteria
root = ElementTree(None, os.path.join(d, PACKAGE_FILE))
is_metapackage = root.find('./export/metapackage') is not None
if ((manifest_name == STACK_FILE and is_metapackage)
or (manifest_name == MANIFEST_FILE and not is_metapackage)
or manifest_name == PACKAGE_FILE):
resource_name = root.findtext('name').strip(' \n\r\t')
if resource_name not in resources:
resources.append(resource_name)
if cache is not None:
cache[resource_name] = d
del dirs[:]
continue # leaf
if manifest_name in files:
resource_name = basename(d)
if resource_name not in resources:
resources.append(resource_name)
if cache is not None:
cache[resource_name] = d
del dirs[:]
continue # leaf
elif MANIFEST_FILE in files or PACKAGE_FILE in files:
# noop if manifest_name==MANIFEST_FILE, but a good
# optimization for stacks.
del dirs[:]
continue # leaf
elif 'rospack_nosubdirs' in files:
del dirs[:]
continue # leaf
# remove hidden dirs (esp. .svn/.git)
[dirs.remove(di) for di in dirs if di[0] == '.']
return resources
def getPlaceName(latitude, longitude):
url = "http://ws.geonames.org/findNearbyPlaceName?lat=%s&lng=%s" % \
(latitude, longitude)
cached_name = name_cache.get(url)
if cached_name:
return cached_name
request = get(url)
tree = ElementTree()
tree.parse(StringIO(request.text.encode('utf-8')))
name = tree.find('geoname/toponymName').text
name_cache.set(url, name)
return name
def read_data(folder):
data = dict()
folder = os.path.abspath(folder)
for subfolder in [os.path.join(folder, x) for x in os.listdir(folder)
if x not in [".", ".."] and os.path.isdir(os.path.join(folder, x))]:
for f in [os.path.join(subfolder, x) for x in os.listdir(subfolder) if x not in [".", ".."]]:
data[f] = dict()
tree = ElementTree()
tree.parse(f)
body = tree.find('text').find('body')
for p in body:
data[f][p.get('n')] = p.text
return data
class etree(ebase):
def __init__( self, fname ):
ebase.__init__( self )
self.et = ElementTree( file=fname )
def write( self, fname ):
self.et.write(fname)
def ensure_child( self, tag ):
retval = self.et.find("./"+tag)
if not retval is None:
return elem( retval )
else:
return elem( SubElement( self.et.getroot(), tag ) )
def __init__(self):
self._faces = OrderedDict()
tree = ElementTree(file=path.face_path + "face.xml")
category = ""
for face in tree.iterfind("./FACEINFO/"):
assert face.tag == "FACE"
face = FaceItem(face)
if category != face.category:
category = face.category
self._faces[category] = OrderedDict()
else:
self._faces[category][face.name] = face
size = tree.find("./WNDCONFIG/Align")
self._col, self._row = int(size.get("Col")), int(size.get("Row"))
self._all_faces_cache = []
def __init__(self):
self._faces = OrderedDict()
tree = ElementTree(file=path.face_path + "face.xml")
category = ""
for face in tree.iterfind("./FACEINFO/"):
assert face.tag == "FACE"
face = FaceItem(face)
if category != face.category:
category = face.category
self._faces[category] = OrderedDict()
else:
self._faces[category][face.name] = face
size = tree.find("./WNDCONFIG/Align")
self._col, self._row = int(size.get("Col")), int(size.get("Row"))
self._all_faces_cache = []
def __init__(self, result_xml):
et = ElementTree()
try:
w = result_xml.encode('utf-8')
def tx(a): return (type(a), len(a))
et.parse(StringIO(w))
except SyntaxError as e:
ptb = traceback.extract_stack()
raise SolrError(e, result_xml, traceback.format_list(ptb))
range_info = et.find('info').find('range_info')
def gn(tagname):
return int(range_info.findtext(tagname))
self.total_results = gn('total_nbr')
self.begin = gn('begin')
self.end = gn('end')
self.results_this_page = gn('contained_in_this_set')
self.result_list = list(str(a.text) \
for a in et.getiterator('identifier'))
def __init__(self, result_xml):
et = ElementTree()
try:
w = result_xml.encode('utf-8')
def tx(a): return (type(a), len(a))
et.parse(StringIO(w))
except SyntaxError as e:
ptb = traceback.extract_stack()
raise SolrError(e, result_xml, traceback.format_list(ptb))
range_info = et.find('info').find('range_info')
def gn(tagname):
return int(range_info.findtext(tagname))
self.total_results = gn('total_nbr')
self.begin = gn('begin')
self.end = gn('end')
self.results_this_page = gn('contained_in_this_set')
self.result_list = list(str(a.text) \
for a in et.getiterator('identifier'))
class GEXFReader(GEXF):
# Class to read GEXF format files
# use read_gexf() function
def __init__(self, node_type=None, version='1.1draft'):
try:
import xml.etree.ElementTree
except ImportError:
raise ImportError('GEXF reader requires '
'xml.elementtree.ElementTree.')
self.node_type = node_type
# assume simple graph and test for multigraph on read
self.simple_graph = True
self.set_version(version)
def __call__(self, stream):
self.xml = ElementTree(file=stream)
g = self.xml.find('{%s}graph' % self.NS_GEXF)
if g is not None:
return self.make_graph(g)
# try all the versions
for version in self.versions:
self.set_version(version)
g = self.xml.find('{%s}graph' % self.NS_GEXF)
if g is not None:
return self.make_graph(g)
raise nx.NetworkXError('No <graph> element in GEXF file.')
def make_graph(self, graph_xml):
# start with empty DiGraph or MultiDiGraph
edgedefault = graph_xml.get('defaultedgetype', None)
if edgedefault == 'directed':
G = nx.MultiDiGraph()
else:
G = nx.MultiGraph()
# graph attributes
graph_name = graph_xml.get('name', '')
if graph_name != '':
G.graph['name'] = graph_name
graph_start = graph_xml.get('start')
if graph_start is not None:
G.graph['start'] = graph_start
graph_end = graph_xml.get('end')
if graph_end is not None:
G.graph['end'] = graph_end
graph_mode = graph_xml.get('mode', '')
if graph_mode == 'dynamic':
G.graph['mode'] = 'dynamic'
else:
G.graph['mode'] = 'static'
# timeformat
self.timeformat = graph_xml.get('timeformat')
if self.timeformat == 'date':
self.timeformat = 'string'
# node and edge attributes
attributes_elements = graph_xml.findall('{%s}attributes' % self.NS_GEXF)
# dictionaries to hold attributes and attribute defaults
node_attr = {}
node_default = {}
edge_attr = {}
edge_default = {}
for a in attributes_elements:
attr_class = a.get('class')
if attr_class == 'node':
na,nd = self.find_gexf_attributes(a)
node_attr.update(na)
node_default.update(nd)
G.graph['node_default'] = node_default
elif attr_class == 'edge':
ea,ed = self.find_gexf_attributes(a)
edge_attr.update(ea)
edge_default.update(ed)
G.graph['edge_default'] = edge_default
else:
raise # unknown attribute class
# Hack to handle Gephi0.7beta bug
# add weight attribute
ea = {'weight': {'type': 'double', 'mode': 'static', 'title': 'weight'}}
ed = {}
edge_attr.update(ea)
edge_default.update(ed)
G.graph['edge_default'] = edge_default
# add nodes
nodes_element = graph_xml.find('{%s}nodes' % self.NS_GEXF)
if nodes_element is not None:
for node_xml in nodes_element.findall('{%s}node' % self.NS_GEXF):
self.add_node(G, node_xml, node_attr)
# add edges
edges_element = graph_xml.find('{%s}edges' % self.NS_GEXF)
if edges_element is not None:
for edge_xml in edges_element.findall('{%s}edge' % self.NS_GEXF):
self.add_edge(G, edge_xml, edge_attr)
# switch to Graph or DiGraph if no parallel edges were found.
if self.simple_graph:
if G.is_directed():
G = nx.DiGraph(G)
else:
G = nx.Graph(G)
return G
def add_node(self, G, node_xml, node_attr, node_pid=None):
# add a single node with attributes to the graph
# get attributes and subattributues for node
data = self.decode_attr_elements(node_attr, node_xml)
data = self.add_parents(data, node_xml) # add any parents
if self.version == '1.1':
data = self.add_slices(data, node_xml) # add slices
else:
data = self.add_spells(data, node_xml) # add spells
data = self.add_viz(data, node_xml) # add viz
data = self.add_start_end(data, node_xml) # add start/end
# find the node id and cast it to the appropriate type
node_id = node_xml.get('id')
if self.node_type is not None:
node_id = self.node_type(node_id)
# every node should have a label
node_label = node_xml.get('label')
data['label'] = node_label
# parent node id
node_pid = node_xml.get('pid', node_pid)
if node_pid is not None:
data['pid'] = node_pid
# check for subnodes, recursive
subnodes = node_xml.find('{%s}nodes' % self.NS_GEXF)
if subnodes is not None:
for node_xml in subnodes.findall('{%s}node' % self.NS_GEXF):
self.add_node(G, node_xml, node_attr, node_pid=node_id)
G.add_node(node_id, data)
def add_start_end(self, data, xml):
# start and end times
ttype = self.timeformat
node_start = xml.get('start')
if node_start is not None:
data['start'] = self.python_type[ttype](node_start)
node_end = xml.get('end')
if node_end is not None:
data['end'] = self.python_type[ttype](node_end)
return data
def add_viz(self, data, node_xml):
# add viz element for node
viz = {}
color = node_xml.find('{%s}color' % self.NS_VIZ)
if color is not None:
if self.VERSION == '1.1':
viz['color'] = {'r': int(color.get('r')),
'g': int(color.get('g')),
'b': int(color.get('b'))}
else:
viz['color'] = {'r': int(color.get('r')),
'g': int(color.get('g')),
'b': int(color.get('b')),
'a': float(color.get('a', 1))}
size=node_xml.find('{%s}size' % self.NS_VIZ)
if size is not None:
viz['size'] = float(size.get('value'))
thickness = node_xml.find('{%s}thickness' % self.NS_VIZ)
if thickness is not None:
viz['thickness'] = float(thickness.get('value'))
shape = node_xml.find('{%s}shape' % self.NS_VIZ)
if shape is not None:
viz['shape'] = shape.get('shape')
if viz['shape'] == 'image':
viz['shape'] = shape.get('uri')
position = node_xml.find('{%s}position' % self.NS_VIZ)
if position is not None:
viz['position'] = {'x': float(position.get('x', 0)),
'y': float(position.get('y', 0)),
'z': float(position.get('z', 0))}
if len(viz) > 0:
data['viz'] = viz
return data
def add_parents(self, data, node_xml):
parents_element = node_xml.find('{%s}parents' % self.NS_GEXF)
if parents_element is not None:
data['parents'] = []
for p in parents_element.findall('{%s}parent' % self.NS_GEXF):
parent = p.get('for')
data['parents'].append(parent)
return data
def add_slices(self, data, node_or_edge_xml):
slices_element = node_or_edge_xml.find('{%s}slices' % self.NS_GEXF)
if slices_element is not None:
data['slices'] = []
for s in slices_element.findall('{%s}slice' % self.NS_GEXF):
start = s.get('start')
end = s.get('end')
data['slices'].append((start, end))
return data
def add_spells(self, data, node_or_edge_xml):
spells_element=node_or_edge_xml.find('{%s}spells' % self.NS_GEXF)
if spells_element is not None:
data['spells'] = []
ttype = self.timeformat
for s in spells_element.findall('{%s}spell' % self.NS_GEXF):
start = self.python_type[ttype](s.get('start'))
end = self.python_type[ttype](s.get('end'))
data['spells'].append((start, end))
return data
def add_edge(self, G, edge_element, edge_attr):
# add an edge to the graph
# raise error if we find mixed directed and undirected edges
edge_direction = edge_element.get('type')
if G.is_directed() and edge_direction == 'undirected':
raise nx.NetworkXError(\
'Undirected edge found in directed graph.')
if (not G.is_directed()) and edge_direction == 'directed':
raise nx.NetworkXError(\
'Directed edge found in undirected graph.')
# Get source and target and recast type if required
source = edge_element.get('source')
target = edge_element.get('target')
if self.node_type is not None:
source = self.node_type(source)
target = self.node_type(target)
data = self.decode_attr_elements(edge_attr, edge_element)
data = self.add_start_end(data, edge_element)
if self.version == '1.1':
data = self.add_slices(data, edge_element) # add slices
else:
data = self.add_spells(data, edge_element) # add spells
# GEXF stores edge ids as an attribute
# NetworkX uses them as keys in multigraphs
# if networkx_key is not specified as an attribute
edge_id = edge_element.get('id')
if edge_id is not None:
data['id'] = edge_id
# check if there is a 'multigraph_key' and use that as edge_id
multigraph_key = data.pop('networkx_key', None)
if multigraph_key is not None:
edge_id = multigraph_key
weight = edge_element.get('weight')
if weight is not None:
data['weight'] = float(weight)
edge_label = edge_element.get('label')
if edge_label is not None:
data['label'] = edge_label
if G.has_edge(source,target):
# seen this edge before - this is a multigraph
self.simple_graph = False
G.add_edge(source, target, key=edge_id, **data)
if edge_direction == 'mutual':
G.add_edge(target, source, key=edge_id, **data)
def decode_attr_elements(self, gexf_keys, obj_xml):
# Use the key information to decode the attr XML
attr = {}
# look for outer '<attvalues>' element
attr_element=obj_xml.find('{%s}attvalues' % self.NS_GEXF)
if attr_element is not None:
# loop over <attvalue> elements
for a in attr_element.findall('{%s}attvalue' % self.NS_GEXF):
key = a.get('for') # for is required
try: # should be in our gexf_keys dictionary
title = gexf_keys[key]['title']
except KeyError:
raise nx.NetworkXError('No attribute defined for=%s.' % key)
atype = gexf_keys[key]['type']
value = a.get('value')
if atype == 'boolean':
value = self.convert_bool[value]
else:
value = self.python_type[atype](value)
if gexf_keys[key]['mode'] == 'dynamic':
# for dynamic graphs use list of three-tuples
# [(value1,start1,end1), (value2,start2,end2), etc]
ttype = self.timeformat
start = self.python_type[ttype](a.get('start'))
end = self.python_type[ttype](a.get('end'))
if title in attr:
attr[title].append((value, start, end))
else:
attr[title] = [(value,start,end)]
else:
# for static graphs just assign the value
attr[title] = value
return attr
def find_gexf_attributes(self, attributes_element):
# Extract all the attributes and defaults
attrs = {}
defaults = {}
mode = attributes_element.get('mode')
for k in attributes_element.findall('{%s}attribute' % self.NS_GEXF):
attr_id = k.get('id')
title = k.get('title')
atype = k.get('type')
attrs[attr_id] = {'title': title, 'type': atype, 'mode': mode}
# check for the 'default' subelement of key element and add
default = k.find('{%s}default' % self.NS_GEXF)
if default is not None:
if atype == 'boolean':
value = self.convert_bool[default.text]
else:
value = self.python_type[atype](default.text)
defaults[title] = value
return attrs, defaults
class GEXFReader(GEXF):
# Class to read GEXF format files
# use read_gexf() function
def __init__(self, node_type=None, version='1.1draft'):
try:
import xml.etree.ElementTree
except ImportError:
raise ImportError('GEXF reader requires '
'xml.elementtree.ElementTree')
self.node_type = node_type
# assume simple graph and test for multigraph on read
self.simple_graph = True
self.set_version(version)
def __call__(self, stream):
self.xml = ElementTree(file=stream)
g = self.xml.find("{%s}graph" % self.NS_GEXF)
if g is not None:
return self.make_graph(g)
# try all the versions
for version in self.versions:
self.set_version(version)
g = self.xml.find("{%s}graph" % self.NS_GEXF)
if g is not None:
return self.make_graph(g)
raise nx.NetworkXError("No <graph> element in GEXF file")
def make_graph(self, graph_xml):
# mode is "static" or "dynamic"
graph_mode = graph_xml.get("mode", "")
self.dynamic = (graph_mode == 'dynamic')
# start with empty DiGraph or MultiDiGraph
edgedefault = graph_xml.get("defaultedgetype", None)
if edgedefault == 'directed':
G = nx.MultiDiGraph()
else:
G = nx.MultiGraph()
# graph attributes
graph_start = graph_xml.get('start')
if graph_start is not None:
G.graph['start'] = graph_start
graph_end = graph_xml.get('end')
if graph_end is not None:
G.graph['end'] = graph_end
# node and edge attributes
attributes_elements = graph_xml.findall("{%s}attributes" %
self.NS_GEXF)
# dictionaries to hold attributes and attribute defaults
node_attr = {}
node_default = {}
edge_attr = {}
edge_default = {}
for a in attributes_elements:
attr_class = a.get("class")
if attr_class == 'node':
na, nd = self.find_gexf_attributes(a)
node_attr.update(na)
node_default.update(nd)
G.graph['node_default'] = node_default
elif attr_class == 'edge':
ea, ed = self.find_gexf_attributes(a)
edge_attr.update(ea)
edge_default.update(ed)
G.graph['edge_default'] = edge_default
else:
raise # unknown attribute class
# Hack to handle Gephi0.7beta bug
# add weight attribute
ea = {
'weight': {
'type': 'double',
'mode': 'static',
'title': 'weight'
}
}
ed = {}
edge_attr.update(ea)
edge_default.update(ed)
G.graph['edge_default'] = edge_default
# add nodes
nodes_element = graph_xml.find("{%s}nodes" % self.NS_GEXF)
if nodes_element is not None:
for node_xml in nodes_element.findall("{%s}node" % self.NS_GEXF):
self.add_node(G, node_xml, node_attr)
# add edges
edges_element = graph_xml.find("{%s}edges" % self.NS_GEXF)
if edges_element is not None:
for edge_xml in edges_element.findall("{%s}edge" % self.NS_GEXF):
self.add_edge(G, edge_xml, edge_attr)
# switch to Graph or DiGraph if no parallel edges were found.
if self.simple_graph:
if G.is_directed():
G = nx.DiGraph(G)
else:
G = nx.Graph(G)
return G
def add_node(self, G, node_xml, node_attr, node_pid=None):
# add a single node with attributes to the graph
# get attributes and subattributues for node
data = self.decode_attr_elements(node_attr, node_xml)
data = self.add_parents(data, node_xml) # add any parents
if self.version == '1.1':
data = self.add_slices(data, node_xml) # add slices
else:
data = self.add_spells(data, node_xml) # add spells
data = self.add_viz(data, node_xml) # add viz
data = self.add_start_end(data, node_xml) # add start/end
# find the node id and cast it to the appropriate type
node_id = node_xml.get("id")
if self.node_type is not None:
node_id = self.node_type(node_id)
# every node should have a label
node_label = node_xml.get("label")
data['label'] = node_label
# parent node id
node_pid = node_xml.get("pid", node_pid)
if node_pid is not None:
data['pid'] = node_pid
# check for subnodes, recursive
subnodes = node_xml.find("{%s}nodes" % self.NS_GEXF)
if subnodes is not None:
for node_xml in subnodes.findall("{%s}node" % self.NS_GEXF):
self.add_node(G, node_xml, node_attr, node_pid=node_id)
G.add_node(node_id, data)
def add_start_end(self, data, xml):
# start and end times
node_start = xml.get("start")
if node_start is not None:
data['start'] = node_start
node_end = xml.get("end")
if node_end is not None:
data['end'] = node_end
return data
def add_viz(self, data, node_xml):
# add viz element for node
viz = {}
color = node_xml.find("{%s}color" % self.NS_VIZ)
if color is not None:
if self.VERSION == '1.1':
viz['color'] = {
'r': int(color.get('r')),
'g': int(color.get('g')),
'b': int(color.get('b'))
}
else:
viz['color'] = {
'r': int(color.get('r')),
'g': int(color.get('g')),
'b': int(color.get('b')),
'a': float(color.get('a', 1)),
}
size = node_xml.find("{%s}size" % self.NS_VIZ)
if size is not None:
viz['size'] = float(size.get('value'))
thickness = node_xml.find("{%s}thickness" % self.NS_VIZ)
if thickness is not None:
viz['thickness'] = float(thickness.get('value'))
shape = node_xml.find("{%s}shape" % self.NS_VIZ)
if shape is not None:
viz['shape'] = shape.get('shape')
if viz['shape'] == 'image':
viz['shape'] = shape.get('uri')
position = node_xml.find("{%s}position" % self.NS_VIZ)
if position is not None:
viz['position'] = {
'x': float(position.get('x', 0)),
'y': float(position.get('y', 0)),
'z': float(position.get('z', 0))
}
if len(viz) > 0:
data['viz'] = viz
return data
def add_parents(self, data, node_xml):
parents_element = node_xml.find("{%s}parents" % self.NS_GEXF)
if parents_element is not None:
data['parents'] = []
for p in parents_element.findall("{%s}parent" % self.NS_GEXF):
parent = p.get('for')
data['parents'].append(parent)
return data
def add_slices(self, data, node_xml):
slices_element = node_xml.find("{%s}slices" % self.NS_GEXF)
if slices_element is not None:
data['slices'] = []
for s in slices_element.findall("{%s}slice" % self.NS_GEXF):
start = s.get('start')
end = s.get('end')
data['slices'].append((start, end))
return data
def add_spells(self, data, node_xml):
spells_element = node_xml.find("{%s}spells" % self.NS_GEXF)
if spells_element is not None:
data['spells'] = []
for s in spells_element.findall("{%s}spell" % self.NS_GEXF):
start = s.get('start')
end = s.get('end')
data['spells'].append((start, end))
return data
def add_edge(self, G, edge_element, edge_attr):
# add an edge to the graph
# raise error if we find mixed directed and undirected edges
edge_direction = edge_element.get("type")
if G.is_directed() and edge_direction == 'undirected':
raise nx.NetworkXError(\
"Undirected edge found in directed graph.")
if (not G.is_directed()) and edge_direction == 'directed':
raise nx.NetworkXError(\
"Directed edge found in undirected graph.")
# Get source and target and recast type if required
source = edge_element.get("source")
target = edge_element.get("target")
if self.node_type is not None:
source = self.node_type(source)
target = self.node_type(target)
data = self.decode_attr_elements(edge_attr, edge_element)
data = self.add_start_end(data, edge_element)
if self.version == '1.1':
data = self.add_slices(data, edge_element) # add slices
else:
data = self.add_spells(data, edge_element) # add spells
# GEXF stores edge ids as an attribute
# NetworkX uses them as keys in multigraphs
# if networkx_key is not specified as an attribute
edge_id = edge_element.get("id")
if edge_id is not None:
data["id"] = edge_id
weight = edge_element.get('weight')
if weight is not None:
data['weight'] = float(weight)
edge_label = edge_element.get("label")
if edge_label is not None:
data['label'] = edge_label
if G.has_edge(source, target) or source == target:
# seen this edge before - this is a multigraph
self.simple_graph = False
G.add_edge(source, target, **data)
if edge_direction == 'mutual':
G.add_edge(target, source, **data)
def decode_attr_elements(self, gexf_keys, obj_xml):
# Use the key information to decode the attr XML
attr = {}
# look for outer "<attvalues>" element
attr_element = obj_xml.find("{%s}attvalues" % self.NS_GEXF)
if attr_element is not None:
# loop over <attvalue> elements
for a in attr_element.findall("{%s}attvalue" % self.NS_GEXF):
key = a.get('for') # for is required
try: # should be in our gexf_keys dictionary
title = gexf_keys[key]['title']
except KeyError:
raise nx.NetworkXError("No attribute defined for=%s" % key)
atype = gexf_keys[key]['type']
value = a.get('value')
if atype == 'boolean':
value = self.convert_bool[value]
else:
value = self.python_type[atype](value)
if gexf_keys[key]['mode'] == 'dynamic':
# for dynamic graphs use list of three-tuples
# [(value1,start1,end1), (value2,start2,end2), etc]
start = a.get('start')
end = a.get('end')
if title in attr:
attr[title].append((value, start, end))
else:
attr[title] = [(value, start, end)]
else:
# for static graphs just assign the value
attr[title] = value
return attr
def find_gexf_attributes(self, attributes_element):
# Extract all the attributes and defaults
attrs = {}
defaults = {}
mode = attributes_element.get('mode')
for k in attributes_element.findall("{%s}attribute" % self.NS_GEXF):
attr_id = k.get("id")
title = k.get('title')
atype = k.get('type')
attrs[attr_id] = {'title': title, 'type': atype, 'mode': mode}
# check for the "default" subelement of key element and add
default = k.find("{%s}default" % self.NS_GEXF)
if default is not None:
if atype == 'boolean':
value = self.convert_bool[default.text]
else:
value = self.python_type[atype](default.text)
defaults[title] = value
return attrs, defaults
class DatabaseTest(unittest.TestCase):
def setUp(self):
self.tree = ElementTree(file=PROVIDERS_PATH)
self.countries_xml = self.tree.findall('country')
db = ServiceProvidersDatabase()
countries = db.get_countries()
self.codes = [country.code for country in countries]
def find_country_idx(self, country_code):
return self.codes.index(country_code)
def test_country_list(self):
db = ServiceProvidersDatabase()
country_list = db.get_countries()
self.assertEqual(len(country_list), len(self.countries_xml))
def test_provider_list(self):
country_code = 'es'
country = self.find_country_idx(country_code)
providers_xml = self.tree.find('country[@code="%s"]' % country_code)
providers_xml = providers_xml.findall('provider')
db = ServiceProvidersDatabase()
db.set_country(country)
providers_list = db.get_providers()
self.assertEqual(len(providers_list), len(providers_xml))
def test_provider_details(self):
country_code = 'es'
country = self.find_country_idx(country_code)
provider = 1
provider_xml = self.tree.find('country[@code="%s"]' % country_code)
provider_xml = provider_xml.find('provider[%s]' % (provider + 1))
db = ServiceProvidersDatabase()
db.set_country(country)
db.set_provider(provider)
provider = db.get_provider()
self.assertTrue(provider.name, provider_xml.find('name').text)
def test_plan_list(self):
country_code = 'es'
country = self.find_country_idx(country_code)
provider = 1
plans_xml = self.tree.find('country[@code="%s"]' % country_code)
plans_xml = plans_xml.find('provider[%s]' % (provider + 1))
plans_xml = plans_xml.findall('.//apn')
db = ServiceProvidersDatabase()
db.set_country(country)
db.set_provider(provider)
plans_list = db.get_plans()
self.assertEqual(len(plans_list), len(plans_xml))
def test_plan_details(self):
country_code = 'es'
country = self.find_country_idx(country_code)
provider = 1
plan = 0
plan_xml = self.tree.find('country[@code="%s"]' % country_code)
plan_xml = plan_xml.find('provider[%s]' % (provider + 1))
plan_xml = plan_xml.find('.//apn[%s]' % (plan + 1))
apn = plan_xml.attrib['value']
db = ServiceProvidersDatabase()
db.set_country(country)
db.set_provider(provider)
db.set_plan(plan)
plan = db.get_plan()
self.assertTrue(apn, plan.apn)
def test_go_through_all_options_from_xml(self):
db = ServiceProvidersDatabase()
for country_el in self.tree.findall('country'):
country_code = country_el.attrib['code']
country_idx = self.find_country_idx(country_code)
db.set_country(country_idx)
country = db.get_country()
self.assertEqual(country_code, country.code)
provider_els = [p_el for p_el in country_el.findall('provider')
if p_el.find('.//gsm')]
if provider_els == []:
self.assertIsNone(db.get_provider())
for provider_idx, provider_el in enumerate(provider_els):
db.set_provider(provider_idx)
provider = db.get_provider()
self.assertEqual(provider_el.find('name').text,
provider.name)
plan_els = provider_el.findall('.//apn')
if plan_els == []:
self.assertIsNone(db.get_plan())
for plan_idx, plan_el in enumerate(plan_els):
apn = plan_el.attrib['value']
db.set_plan(plan_idx)
plan = db.get_plan()
if plan:
self.assertEqual(apn, plan.apn)
def test_go_trough_all_combo_options(self):
db = ServiceProvidersDatabase()
countries = db.get_countries()
for country in countries:
db.set_country(country.idx)
new_country = db.get_country()
self.assertEqual(country.code, new_country.code)
providers = db.get_providers()
for provider in providers:
db.set_provider(provider.idx)
new_provider = db.get_provider()
self.assertEqual(provider.name, new_provider.name)
plans = db.get_plans()
for plan in plans:
db.set_plan(plan.idx)
new_plan = db.get_plan()
self.assertEqual(plan.name, new_plan.name)
def __init__(self,
protocol = None,
host_name = None,
user_name = None,
password = None,
database_configuration = None,
test = False,
database_server_configuration_file_path = None,
sqlite_db_path = None,
blob_compression = False):
if database_server_configuration_file_path is None:
database_server_configuration_file_path = paths.get_opus_home_path('settings', 'database_server_configurations.xml')
if (protocol is None or test) and host_name is None and user_name is None and password is None:
if not os.path.exists(database_server_configuration_file_path):
raise Exception('You do not have a file %s storing information about your database server configurations. Cannot load database.'%database_server_configuration_file_path)
if database_configuration is None:
db_node = self._database_configuration_node()
else:
db_node = database_configuration
database_configuration = ElementTree(file = database_server_configuration_file_path).getroot().find(db_node)
if database_configuration is None:
raise Exception('Could not find an entry in %s for %s. Cannot load database.'%(database_server_configuration_file_path, db_node))
self.protocol = database_configuration.find(self.PROTOCOL_TAG).text
self.host_name = database_configuration.find(self.HOST_NAME_TAG).text
self.user_name = database_configuration.find(self.USER_NAME_TAG).text
self.password = database_configuration.find(self.PASSWORD_TAG).text
blob_compression = database_configuration.find(self.BLOB_COMPRESSION_TAG)
if blob_compression != None and blob_compression.text == "True":
blob_compression = True
else:
blob_compression = False
else:
if protocol is None:
self.protocol = get_default_database_engine()
else:
self.protocol = protocol.lower()
if host_name is None:
self.host_name = 'localhost'
else:
self.host_name = host_name
if user_name is None:
self.user_name = ''
else:
self.user_name = user_name
if password is None:
self.password = ''
else:
self.password = password
# If the password is the empty string or None, check if it is defined in the environment variable
# SQLPASSWORD - if so, use that.
if (self.password is None or self.password=='') and 'SQLPASSWORD' in os.environ:
self.password = os.environ['SQLPASSWORD']
self.sqlite_db_path = sqlite_db_path
self.blob_compression = blob_compression
class GEXFReader(GEXF):
# Class to read GEXF format files
# use read_gexf() function
def __init__(self, node_type=None):
try:
import xml.etree.cElementTree
except ImportError:
raise ImportError("GEXF reader requires", "xml.elementtree.ElementTree")
self.node_type = node_type
# assume simple graph and test for multigraph on read
self.simple_graph = True
def __call__(self, stream):
self.xml = ElementTree(file=stream)
g = self.xml.find("{%s}graph" % self.NS_GEXF)
if g is None:
raise nx.NetworkXError("No <graph> element in GEXF file")
return self.make_graph(g)
def make_graph(self, graph_xml):
# mode is "static" or "dynamic"
graph_mode = graph_xml.get("mode", "")
self.dynamic = graph_mode == "dynamic"
# start with empty DiGraph or MultiDiGraph
edgedefault = graph_xml.get("defaultedgetype", None)
if edgedefault == "directed":
G = nx.MultiDiGraph()
else:
G = nx.MultiGraph()
# graph attributes
graph_start = graph_xml.get("start")
if graph_start is not None:
G.graph["start"] = graph_start
graph_end = graph_xml.get("end")
if graph_end is not None:
G.graph["end"] = graph_end
# node and edge attributes
attributes_elements = graph_xml.findall("{%s}attributes" % self.NS_GEXF)
# dictionaries to hold attributes and attribute defaults
node_attr = {}
node_default = {}
edge_attr = {}
edge_default = {}
for a in attributes_elements:
attr_class = a.get("class")
if attr_class == "node":
na, nd = self.find_gexf_attributes(a)
node_attr.update(na)
node_default.update(nd)
G.graph["node_default"] = node_default
elif attr_class == "edge":
ea, ed = self.find_gexf_attributes(a)
edge_attr.update(ea)
edge_default.update(ed)
G.graph["edge_default"] = edge_default
else:
raise # unknown attribute class
# add nodes
nodes_element = graph_xml.find("{%s}nodes" % self.NS_GEXF)
if nodes_element is not None:
for node_xml in nodes_element.findall("{%s}node" % self.NS_GEXF):
self.add_node(G, node_xml, node_attr)
# add edges
edges_element = graph_xml.find("{%s}edges" % self.NS_GEXF)
if edges_element is not None:
for edge_xml in edges_element.findall("{%s}edge" % self.NS_GEXF):
self.add_edge(G, edge_xml, edge_attr)
# switch to Graph or DiGraph if no parallel edges were found.
if self.simple_graph:
if G.is_directed():
G = nx.DiGraph(G)
else:
G = nx.Graph(G)
return G
def add_node(self, G, node_xml, node_attr, node_pid=None):
# add a single node with attributes to the graph
# get attributes and subattributues for node
data = self.decode_attr_elements(node_attr, node_xml)
data = self.add_parents(data, node_xml) # add any parents
data = self.add_slices(data, node_xml) # add slices
data = self.add_viz(data, node_xml) # add viz
data = self.add_start_end(data, node_xml) # add start/end
# find the node id and cast it to the appropriate type
node_id = node_xml.get("id")
if self.node_type is not None:
node_id = self.node_type(node_id)
# every node should have a label
node_label = node_xml.get("label")
data["label"] = node_label
# parent node id
node_pid = node_xml.get("pid", node_pid)
if node_pid is not None:
data["pid"] = node_pid
# check for subnodes, recursive
subnodes = node_xml.find("{%s}nodes" % self.NS_GEXF)
if subnodes is not None:
for node_xml in subnodes.findall("{%s}node" % self.NS_GEXF):
self.add_node(G, node_xml, node_attr, node_pid=node_id)
G.add_node(node_id, data)
def add_start_end(self, data, xml):
# start and end times
node_start = xml.get("start")
if node_start is not None:
data["start"] = node_start
node_end = xml.get("end")
if node_end is not None:
data["end"] = node_end
return data
def add_viz(self, data, node_xml):
# add viz element for node
viz = {}
color = node_xml.find("{%s}color" % self.NS_VIZ)
if color is not None:
viz["color"] = {"r": int(color.get("r")), "g": int(color.get("g")), "b": int(color.get("b"))}
size = node_xml.find("{%s}size" % self.NS_VIZ)
if size is not None:
viz["size"] = float(size.get("value"))
thickness = node_xml.find("{%s}thickness" % self.NS_VIZ)
if thickness is not None:
viz["thickness"] = float(thickness.get("value"))
shape = node_xml.find("{%s}shape" % self.NS_VIZ)
if shape is not None:
viz["shape"] = shape.get("shape")
if viz["shape"] == "image":
viz["shape"] = shape.get("uri")
position = node_xml.find("{%s}position" % self.NS_VIZ)
if position is not None:
viz["position"] = {
"x": float(position.get("x", 0)),
"y": float(position.get("y", 0)),
"z": float(position.get("z", 0)),
}
if len(viz) > 0:
data["viz"] = viz
return data
def add_parents(self, data, node_xml):
parents_element = node_xml.find("{%s}parents" % self.NS_GEXF)
if parents_element is not None:
data["parents"] = []
for p in parents_element.findall("{%s}parent" % self.NS_GEXF):
parent = p.get("for")
data["parents"].append(parent)
return data
def add_slices(self, data, node_xml):
slices_element = node_xml.find("{%s}slices" % self.NS_GEXF)
if slices_element is not None:
data["slices"] = []
for s in slices_element.findall("{%s}slice" % self.NS_GEXF):
start = s.get("start")
end = s.get("end")
data["slices"].append((start, end))
return data
def add_edge(self, G, edge_element, edge_attr):
# add an edge to the graph
# raise error if we find mixed directed and undirected edges
edge_direction = edge_element.get("type")
if G.is_directed() and edge_direction == "undirected":
raise nx.NetworkXError("Undirected edge found in directed graph.")
if (not G.is_directed()) and edge_direction == "directed":
raise nx.NetworkXError("Directed edge found in undirected graph.")
# Get source and target and recast type if required
source = edge_element.get("source")
target = edge_element.get("target")
if self.node_type is not None:
source = self.node_type(source)
target = self.node_type(target)
data = self.decode_attr_elements(edge_attr, edge_element)
data = self.add_start_end(data, edge_element)
# GEXF stores edge ids as an attribute
# NetworkX uses them as keys in multigraphs
# if networkx_key is not specified as an attribute
edge_id = edge_element.get("id")
if edge_id is not None:
data["id"] = edge_id
# check if there is a 'multigraph_key' and use that as edge_id
multigraph_key = data.pop("networkx_key", None)
if multigraph_key is not None:
edge_id = multigraph_key
weight = edge_element.get("weight")
if weight is not None:
data["weight"] = float(weight)
edge_label = edge_element.get("label")
if edge_label is not None:
data["label"] = edge_label
if G.has_edge(source, target):
# seen this edge before - this is a multigraph
self.simple_graph = False
G.add_edge(source, target, key=edge_id, **data)
if edge_direction == "mutual":
G.add_edge(target, source, key=edge_id, **data)
def decode_attr_elements(self, gexf_keys, obj_xml):
# Use the key information to decode the attr XML
attr = {}
# look for outer "<attvalues>" element
attr_element = obj_xml.find("{%s}attvalues" % self.NS_GEXF)
if attr_element is not None:
# loop over <attvalue> elements
for a in attr_element.findall("{%s}attvalue" % self.NS_GEXF):
key = a.get("for") # for is required
try: # should be in our gexf_keys dictionary
title = gexf_keys[key]["title"]
except KeyError:
raise nx.NetworkXError("No attribute defined for=%s" % key)
atype = gexf_keys[key]["type"]
value = a.get("value")
if atype == "boolean":
value = self.convert_bool[value]
else:
value = self.python_type[atype](value)
if gexf_keys[key]["mode"] == "dynamic":
# for dynamic graphs use list of three-tuples
# [(value1,start1,end1), (value2,start2,end2), etc]
start = a.get("start")
end = a.get("end")
if title in attr:
attr[title].append((value, start, end))
else:
attr[title] = [(value, start, end)]
else:
# for static graphs just assign the value
attr[title] = value
return attr
def find_gexf_attributes(self, attributes_element):
# Extract all the attributes and defaults
attrs = {}
defaults = {}
mode = attributes_element.get("mode")
for k in attributes_element.findall("{%s}attribute" % self.NS_GEXF):
attr_id = k.get("id")
title = k.get("title")
atype = k.get("type")
attrs[attr_id] = {"title": title, "type": atype, "mode": mode}
# check for the "default" subelement of key element and add
default = k.find("{%s}default" % self.NS_GEXF)
if default is not None:
if atype == "boolean":
value = self.convert_bool[default.text]
else:
value = self.python_type[atype](default.text)
defaults[title] = value
return attrs, defaults
#!/bin/evn python2.6
#!-*- coding:utf-8 -*-
#from xml.etree.cElementTree import ElementTree
try:
from xml.etree.cElementTree import ElementTree
except:
from xml.etree.ElementTree import ElementTree
import sys
tree = ElementTree()
tree.parse("index.xhtml")
p = tree.find("body/p")
print p.iter()
sys.exit()
links = list(p.iter("a"))
sys.exit()
for i in links:
print i
def load(self, diretorio):
"""
Le todo o arquivo para uma variavel do tipo ElementTree
percorre a variavel ElementTree pegando as informacoes
necessarias para montar o automato
"""
# pegando o diretorio base
# base_path = os.path.dirname(os.path.realpath(__file__))
# base_path = base_path + '\\' + diretorio
# diretorio = base_path
# lendo o arquivo xml tree usando elementtree
self.__init__()
tree = ElementTree() # instanciando o ElementTree
tree.parse(diretorio) # lendo o conteudo diretorio para dentro de tree
tree._setroot(tree.find("structure/structure")) # setando o root mais adequado
root = tree.getroot() # pegando o root setado
# tipo variavel que vai guarda o conteudo da tag type
for child in root:
tipo = child.attrib['type'] # pegando o conteudo da tag type
if tipo == "start_state": # get the start states
for child2 in child:
for child3 in child2:
if child3.tag == "name":
self.estados_iniciais.append(child3.text) # adicionando elemento a lista de estados iniciais
break
if tipo == "input_alph": # get the alphabet from automat
for child2 in child:
self.alphabet.append(child2.text) # adicionando elemento a lista com as letras do alfabeto
if tipo == "final_states": # get the final states
for child2 in child:
for child3 in child2:
if child3.tag == "name":
self.estados_finais.append(child3.text) # adicionando elemento a lista de estados finais
break
if tipo == "transition_set": # pega as conexoes entre os estados
inp = "" # armazena o input
para = "" # armazena o estado de entrada
de = "" # armazena o estado de saida
flag = "" # variavel que ajuda a
entradas = [] # verificar se um novo dicionario deve ser criado dentro do dicionario principal
for child2 in child:
for child3 in child2:
if child3.tag == "from": # get estado de saida
for child4 in child3:
if (child4.tag == "name"):
de = child4.text
break
if child3.tag == "input": # get input
inp = child3.text
if child3.tag == "to":
for child4 in child3:
if child4.tag == "name": # get estado de entrada
para = child4.text
break
if flag != de: # verifica se o estado de entrada ja foi alocado dentro da estrutura principal
flag = de # atribui o novo estado a flag
self.transitions[de] = {} # aloca um dicionario para dentro do dicionario principal na chave "de"
del entradas[:] # limpa a lista de
try:
self.transitions[de][inp].append(para)
except KeyError:
self.transitions[de][inp] = [] # aloca uma lista para cada letra do alfabeto
self.transitions[de][inp].append(para) # adiciona um elemento ao dicionario
if tipo == "state_set": # pega todos os estados usados no automato
for child2 in child:
for child3 in child2:
if child3.tag == "name":
self.estados.append(child3.text) # adiciona elemento a lista "estados"
break
return Af(self.estados, self.alphabet, self.transitions, self.estados_iniciais, self.estados_finais)
xml_template = "Milwaukee_Template.xml"
output_dir = "xml"
rows = arcpy.SearchCursor(fc)
for row in rows:
geom = row.shape
ex = geom.extent
ex_dict = {"westbc" : ex.XMin,
"southbc" : ex.YMin,
"eastbc" : ex.XMax,
"northbc" : ex.YMax}
# Get the raster name
quad_name = row.getValue("Name_qquad")
# <QQ name>_LULC.xml file path and name
lulc_xml_out = os.path.join(output_dir, str(quad_name) + "_LULC.img.xml")
# Start the XML work
tree = ElementTree()
tree.parse(xml_template)
for k, v in ex_dict.iteritems():
corner = tree.find("idinfo/spdom/bounding/" + k)
corner.text = str(v)
# <QQ name>_LULC.xml
tree.write(lulc_xml_out)
# <QQ name>_TREE.xml
tree_xml_out = lulc_xml_out.replace("_LULC.img", "_TREE.img")
shutil.copy(lulc_xml_out, tree_xml_out)
# <QQ name>.xml
quadname_only_xml_out = os.path.splitext(tree_xml_out)[0][:-9] + "_poly.shp.xml"
shutil.copy(tree_xml_out, quadname_only_xml_out)
def get_pmid_from_doi(doi):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=%s' % doi
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
return tree.find('IdList/Id').text
def __init__(self,
protocol=None,
host_name=None,
user_name=None,
password=None,
database_configuration=None,
test=False,
database_server_configuration_file_path=None,
sqlite_db_path=None,
blob_compression=False):
if database_server_configuration_file_path is None:
database_server_configuration_file_path = paths.get_opus_home_path(
'settings', 'database_server_configurations.xml')
if (protocol is None or test
) and host_name is None and user_name is None and password is None:
if not os.path.exists(database_server_configuration_file_path):
raise Exception(
'You do not have a file %s storing information about your database server configurations. Cannot load database.'
% database_server_configuration_file_path)
if database_configuration is None:
db_node = self._database_configuration_node()
else:
db_node = database_configuration
database_configuration = ElementTree(
file=database_server_configuration_file_path).getroot().find(
db_node)
if database_configuration is None:
raise Exception(
'Could not find an entry in %s for %s. Cannot load database.'
% (database_server_configuration_file_path, db_node))
self.protocol = database_configuration.find(self.PROTOCOL_TAG).text
self.host_name = database_configuration.find(
self.HOST_NAME_TAG).text
self.user_name = database_configuration.find(
self.USER_NAME_TAG).text
self.password = database_configuration.find(self.PASSWORD_TAG).text
blob_compression = database_configuration.find(
self.BLOB_COMPRESSION_TAG)
if blob_compression != None and blob_compression.text == "True":
blob_compression = True
else:
blob_compression = False
else:
if protocol is None:
self.protocol = get_default_database_engine()
else:
self.protocol = protocol.lower()
if host_name is None:
self.host_name = 'localhost'
else:
self.host_name = host_name
if user_name is None:
self.user_name = ''
else:
self.user_name = user_name
if password is None:
self.password = ''
else:
self.password = password
# If the password is the empty string or None, check if it is defined in the environment variable
# SQLPASSWORD - if so, use that.
if (self.password is None
or self.password == '') and 'SQLPASSWORD' in os.environ:
self.password = os.environ['SQLPASSWORD']
self.sqlite_db_path = sqlite_db_path
self.blob_compression = blob_compression
con = sqlite3.connect(DB_FILE, isolation_level=None)
con.execute(sql,
(
channel,
start,
play_time
))
con.close()
sys.exit()
####################################################################################################
xml_file = sys.argv[1]
tree = ElementTree()
tree.parse(xml_file)
el = tree.find("programme")
channel = el.get("channel")
start = int(tree.find("epoch[@type='start']").text)
play_time = 0
signal.signal(signal.SIGHUP, update)
signal.signal(signal.SIGINT, update)
signal.signal(signal.SIGQUIT, update)
signal.signal(signal.SIGTERM, update)
while True:
sleep(10)
play_time += 10