def fulltext_search(self, query, rows=None, start=None):
"""Does an advanced search on fulltext:blah.
You get back a pair (x,y) where x is the total # of hits
and y is a list of identifiers like ["foo", "bar", etc.]"""
query = self._prefix_query('fulltext', query)
result_list = self.raw_search(query, rows=rows, start=start)
e = ElementTree()
try:
e.parse(StringIO(result_list))
except SyntaxError as e:
raise SolrError(e)
total_nbr_text = e.find('info/range_info/total_nbr').text
# total_nbr_text = e.find('result').get('numFound') # for raw xml
total_nbr = int(total_nbr_text) if total_nbr_text else 0
out = []
for r in e.getiterator('hit'):
for d in r.find('metadata'):
for x in list(d.getiterator()):
if x.tag == "identifier":
xid = six.text_type(x.text).encode('utf-8')
if xid.startswith('OCA/'):
xid = xid[4:]
elif xid.endswith('.txt'):
xid = xid.split('/')[-1].split('_')[0]
elif xid.endswith('_ZZ'):
xid = xid[:-3]
out.append(xid)
break
return (total_nbr, out)
def updatebq(self):
from xml.etree.cElementTree import ElementTree
tree = ElementTree()
tree.parse(GSXML)
tvlist = []
for iptv in tree.findall('iptv'):
name = iptv.findtext('name').title()
protocol, serviceType, bufferSize, epgId = iptv.findtext(
'type').split(':')
uri = iptv.findtext('uri')
if protocol in 'livestreamer':
uri = 'http://localhost:88/' + uri
uri = uri.replace(':', '%3a')
service = '#SERVICE {s}:0:1:{e}:{e}:0:0:0:0:0:{u}:{n}\n'.format(
s=serviceType, e=epgId, u=uri, n=name)
tvlist.append((name, service))
tvlist = sorted(tvlist, key=lambda channel: channel[0])
with open(GSBQ, 'w') as f:
f.write('#NAME GreekStreamTV\n')
for name, service in tvlist:
f.write(service)
com = 'cat /usr/lib/enigma2/python/Plugins/Satdreamgr/UpdateBouquet/stream.xml ; rm /usr/lib/enigma2/python/Plugins/Satdreamgr/UpdateBouquet/stream.xml'
out = os.popen(com)
return list
def getEvents(feed):
"""
Creates events from an ATOM feed with GeoRSS points.
"""
events = []
tree = ElementTree()
tree.parse(feed)
entries = ElementTree(tree).iter('{http://www.w3.org/2005/Atom}entry')
for entry in entries:
author = entry.find('{http://www.w3.org/2005/Atom}author')
try:
name = author.find('{http://www.w3.org/2005/Atom}name').text
uri = author.find('{http://www.w3.org/2005/Atom}uri').text
except AttributeError:
continue
try:
point = entry.find('{http://www.georss.org/georss}point').text
latitude = point.split()[0]
longitude = point.split()[1]
except AttributeError:
continue
published = parse_date(
entry.find('{http://www.w3.org/2005/Atom}published').text
)
event = Event(name, uri, published, latitude, longitude)
events.append(event)
return events
def search(query):
query = query.lower()
parsed = expr.parseString(query)[0]
res = parsed()
print
print("Searching for \"{}\"".format(query))
if res[1]:
print("Found {} results".format(len(res[0])))
for file_id in list(res[0])[:5]:
print(files[file_id])
words = parsed.words()
tree = ElementTree()
tree.parse(files[file_id])
body = tree.find('text').find('body')
for p in body:
words_copy = list(words)
at_least_one = False
for word in words_copy:
if word in word_reg_exp.findall(p.text.lower().encode('utf-8')):
at_least_one = True
words.remove(word)
if at_least_one:
print("----| {}".format(p.text.encode('utf-8')))
print
else:
print("Negative requests are not allowed")
print
def list_articles(target_directory, supplementary_materials=False, skip=[]):
"""
Iterates over archive files in target_directory, yielding article information.
"""
listing = listdir(target_directory)
for filename in listing:
with tarfile.open(path.join(target_directory, filename)) as archive:
for item in archive:
if item.name in skip:
continue
if path.splitext(item.name)[1] == '.nxml':
content = archive.extractfile(item)
tree = ElementTree()
tree.parse(content)
result = {}
result['name'] = item.name
result['article-contrib-authors'] = _get_article_contrib_authors(tree)
result['article-title'] = _get_article_title(tree)
result['article-abstract'] = _get_article_abstract(tree)
result['journal-title'] = _get_journal_title(tree)
result['article-date'] = _get_article_date(tree)
result['article-url'] = _get_article_url(tree)
result['article-license-url'] = _get_article_license_url(tree)
result['article-copyright-holder'] = _get_article_copyright_holder(tree)
if supplementary_materials:
result['supplementary-materials'] = _get_supplementary_materials(tree)
yield result
def add_from_file(self, filename):
'''parses xml file and stores wanted details'''
Gtk.Builder.add_from_file(self, filename)
# extract data for the extra interfaces
tree = ElementTree()
tree.parse(filename)
ele_widgets = tree.getiterator("object")
for ele_widget in ele_widgets:
name = ele_widget.attrib['id']
widget = self.get_object(name)
# populate indexes - a dictionary of widgets
self.widgets[name] = widget
# populate a reversed dictionary
self._reverse_widget_dict[widget] = name
# populate connections list
ele_signals = ele_widget.findall("signal")
connections = [(name, ele_signal.attrib['name'],
ele_signal.attrib['handler'])
for ele_signal in ele_signals]
if connections:
self.connections.extend(connections)
ele_signals = tree.getiterator("signal")
for ele_signal in ele_signals:
self.glade_handler_dict.update(
{ele_signal.attrib["handler"]: None})
def __init__(self, name, file, parent=None):
AWindow.__init__(self, name, file, parent)
self.builder = Gtk.Builder()
self.builder.add_from_file(self.file)
self.item = self.builder.get_object(self.name)
if self.parent is not None:
self.item.set_transient_for(self.parent.item)
self.types = {}
for cls in self.classes:
if hasattr(cls, 'type'):
self.types[cls.type] = cls
tree = ElementTree()
tree.parse(self.file)
ele_widgets = tree.getiterator("object")
for ele_widget in ele_widgets:
name = ele_widget.attrib['id']
widget = self.builder.get_object(name)
type = widget.__class__.__name__
if type in self.types:
self.widgets[name] = self.types[type](widget)
else:
self.other_widgets[name] = widget
self.item.connect('delete-event', self.emit_closed)
def _getRoleInfo(base_url, method, acceptable_params, debug, **args):
"""Makes the request and parses the response for its wrapper method"""
#check if acceptable criteria was entered (prevents some errors)
if not _correct_params(acceptable_params,**args):
raise ValueError("At least one parameter with incorrect name given. Check http://devel.yahoo.com/igor/guide/rest_api.html for acceptable parameters")
response = _GET(base_url, method, debug, **args)
# parse the XML response
tree = ElementTree()
tree.parse(response)
role_dic = {}
members = []
rules = []
for elem in tree.iter():
if elem.tag == "role":
role_dic.update(elem.attrib)
if elem.tag == "host":
members.append( elem.attrib["name"] )
if elem.tag == "rule":
rules.append( elem.text )
role_dic["members"] = members
role_dic["rules"] = rules
return Obj(role_dic)
def fulltext_search(self, query, rows=None, start=None):
"""Does an advanced search on fulltext:blah.
You get back a pair (x,y) where x is the total # of hits
and y is a list of identifiers like ["foo", "bar", etc.]"""
query = self._prefix_query('fulltext', query)
result_list = self.raw_search(query, rows=rows, start=start)
e = ElementTree()
try:
e.parse(StringIO(result_list))
except SyntaxError as e:
raise SolrError(e)
total_nbr_text = e.find('info/range_info/total_nbr').text
# total_nbr_text = e.find('result').get('numFound') # for raw xml
total_nbr = int(total_nbr_text) if total_nbr_text else 0
out = []
for r in e.getiterator('hit'):
for d in r.find('metadata'):
for x in list(d.getiterator()):
if x.tag == "identifier":
xid = six.text_type(x.text).encode('utf-8')
if xid.startswith('OCA/'):
xid = xid[4:]
elif xid.endswith('.txt'):
xid = xid.split('/')[-1].split('_')[0]
elif xid.endswith('_ZZ'):
xid = xid[:-3]
out.append(xid)
break
return (total_nbr, out)
def pagetext_search(self, locator, query, rows=None, start=None):
"""Does an advanced search on
pagetext:blah locator:identifier
where identifier is one of the id's from fulltext search.
You get back a list of page numbers like [21, 25, 39]."""
def extract(page_id):
"""TODO: DjVu format is deprecated. Is this function
still even used?
A page id is something like
'adventsuburbanit00butlrich_0065.djvu',
which this function extracts asa a locator and
a leaf number ('adventsuburbanit00butlrich', 65). """
g = re.search('(.*)_(\d{4})\.djvu$', page_id)
a,b = g.group(1,2)
return a, int(b)
# try using qf= parameter here and see if it gives a speedup. @@
# pdb.set_trace()
query = self._prefix_query('pagetext', query)
page_hits = self.raw_search(query,
fq='locator:' + locator,
rows=rows,
start=start)
XML = ElementTree()
try:
XML.parse(StringIO(page_hits))
except SyntaxError as e:
raise SolrError(e)
page_ids = list(e.text for e in XML.getiterator('identifier'))
return [extract(x)[1] for x in page_ids]
def extractImages(html):
if html is None:
return [], html
tree = ElementTree()
tree.parse(StringIO(html))
imagetags = tree.findall(".//img")
images = []
for tag in imagetags:
image = tag.get('src')
path, name = os.path.split(image)
if image not in images:
images.append(image)
tag.set('alt', name)
tag.set('title', name)
#index files for multipart storage
index = {}
for image in images:
path, name = os.path.split(image)
index[image] = '0x%08x' % binascii.crc32(name)
#update html email image tags
for tag in imagetags:
image = tag.get('src')
tag.set('src', "cid:%s" % index[image])
html = StringIO()
tree.write(html)
html.write("\n")
return [index, html.getvalue()]
class _DXML(object):
def __init__(self, path, tagmap={}, tagdefault=None, **options):
self._path, self._options = path, options
self._tree = ElementTree()
self._tree.parse(self._path)
self.verbosity, self.traceback, self.graceful = 1, False, False
self._tagmap = tagmap
self._tagdefault = self._trivial if tagdefault is None else tagdefault
def __iter__(self):
self._preiter_hook()
# Stage 1: namespaces
for o in self._xml_namespaces(): # IGNORE:E1101
yield o
# Stage 2: resources
r = self._tree.getroot()
for e in [r] + r.getchildren(): # IGNORE:E1101
try:
for o in self._tagmap.get(e.tag, self._tagdefault)(e):
yield o
except Exception, x:
self._except(Exception, x)
# Stage 3: inheritance etc.
self._postiter_hook()
def extract_positions(url):
tree = ElementTree()
parser = XMLParser(encoding="iso-8859-1")
data = urllib.urlopen(url)
tree.parse(data, parser=parser)
positions = tree.getroot().findall("team")
allpos = []
for pos in positions:
realpos = pos.find("pos")
latitude = float(realpos.attrib['a'])
longitude = float(realpos.attrib['o'])
speed = float(realpos.attrib['s'])
course = float(realpos.attrib['c'])
last_update = datetime.utcfromtimestamp(int(realpos.attrib["w"]))
dtf = float(realpos.attrib['d'])
_id = pos.attrib["id"]
# pos = geo.xyz(latitude, latitude)
# final object
result = {}
result["str_latitude"] = format_deg(latitude, "N", "S")
result["str_longitude"] = format_deg(longitude, "E", "W")
result["speed"] = speed
result["course"] = course
result["_id"] = _id
result["dtf"] = dtf
result["last_update"] = last_update
allpos.append(result)
return allpos
def extractElement(self, elementpath, respdata):
try:
tree = ElementTree()
tree.parse(StringIO(respdata))
except:
return None
# Strip off the top-level item
if elementpath[0] == '/':
elementpath = elementpath[1:]
splits = elementpath.split('/', 1)
root = splits[0]
if tree.getroot().tag != root:
return None
elif len(splits) == 1:
return tree.getroot().text
else:
elementpath = splits[1]
e = tree.find(elementpath)
if e is not None:
return e.text
else:
return None
def __get_runtime_status(self, runtime_status_files):
'''
Helper funcion for montioring the status of the task.
It reads the status files of all modules and reports which ones are still
running.
'''
task_status = 'finished'
running_module_slots = list()
for (slot_name, status_file) in runtime_status_files.items():
if not os.path.isfile(status_file):
time.sleep(0.1)
assert os.path.isfile(status_file), 'runtime status file %r not found.' \
% status_file
tree = ElementTree()
tree.parse(status_file)
status_file_xml_root_node = tree.getroot()
node = status_file_xml_root_node.find('status')
status = node.text.strip()
if status == 'running':
task_status = 'running'
running_module_slots.append(slot_name)
return (task_status, running_module_slots)
def list_articles(target_directory, supplementary_materials=False, skip=[]):
listing = listdir(target_directory)
for filename in listing:
result_tree = ElementTree()
result_tree.parse(path.join(target_directory, filename))
for tree in result_tree.iterfind('article'):
pmcid = _get_pmcid(tree)
if pmcid in skip:
continue
result = {}
result['name'] = pmcid
result['doi'] = _get_article_doi(tree)
result['article-categories'] = _get_article_categories(tree)
result['article-contrib-authors'] = _get_article_contrib_authors(
tree)
result['article-title'] = _get_article_title(tree)
result['article-abstract'] = _get_article_abstract(tree)
result['journal-title'] = _get_journal_title(tree)
result['article-year'], \
result['article-month'], \
result['article-day'] = _get_article_date(tree)
result['article-url'] = _get_article_url(tree)
result['article-license-url'], \
result['article-license-text'], \
result['article-copyright-statement'] = _get_article_licensing(tree)
result['article-copyright-holder'] = _get_article_copyright_holder(
tree)
if supplementary_materials:
result[
'supplementary-materials'] = _get_supplementary_materials(
tree)
yield result
def _get_crumb(self):
"""Gets crumb by making the appropriate API call"""
response = urllib2.urlopen("".join([self.base_url, "getCrumb"]))
tree = ElementTree()
tree.parse(response)
for elem in tree.getroot().findall("crumb"):
self.crumb = elem.text
def pagetext_search(self, locator, query, rows=None, start=None):
"""Does an advanced search on
pagetext:blah locator:identifier
where identifier is one of the id's from fulltext search.
You get back a list of page numbers like [21, 25, 39]."""
def extract(page_id):
"""A page id is something like
'adventsuburbanit00butlrich_0065.djvu',
which this function extracts asa a locator and
a leaf number ('adventsuburbanit00butlrich', 65). """
g = re.search('(.*)_(\d{4})\.djvu$', page_id)
a, b = g.group(1, 2)
return a, int(b)
# try using qf= parameter here and see if it gives a speedup. @@
# pdb.set_trace()
query = self._prefix_query('pagetext', query)
page_hits = self.raw_search(query,
fq='locator:' + locator,
rows=rows,
start=start)
XML = ElementTree()
try:
XML.parse(StringIO(page_hits))
except SyntaxError, e:
raise SolrError, e
def add_from_file(self, filename):
'''parses xml file and stores wanted details'''
Gtk.Builder.add_from_file(self, filename)
# extract data for the extra interfaces
tree = ElementTree()
tree.parse(filename)
ele_widgets = tree.getiterator("object")
for ele_widget in ele_widgets:
name = ele_widget.attrib['id']
widget = self.get_object(name)
# populate indexes - a dictionary of widgets
self.widgets[name] = widget
# populate a reversed dictionary
self._reverse_widget_dict[widget] = name
# populate connections list
ele_signals = ele_widget.findall("signal")
connections = [
(name,
ele_signal.attrib['name'],
ele_signal.attrib['handler']) for ele_signal in ele_signals]
if connections:
self.connections.extend(connections)
ele_signals = tree.getiterator("signal")
for ele_signal in ele_signals:
self.glade_handler_dict.update(
{ele_signal.attrib["handler"]: None})
def extractElement(self, elementpath, respdata):
try:
tree = ElementTree()
tree.parse(StringIO(respdata))
except:
return None
# Strip off the top-level item
if elementpath[0] == '/':
elementpath = elementpath[1:]
splits = elementpath.split('/', 1)
root = splits[0]
if tree.getroot().tag != root:
return None
elif len(splits) == 1:
return tree.getroot().text
else:
elementpath = splits[1]
e = tree.find(elementpath)
if e is not None:
return e.text
else:
return None
def _getHostState(base_url, method, acceptable_params, debug, **args):
"""Makes the request and parses the response for its wrapper method"""
#check if acceptable criteria was entered (prevents some errors)
if not _correct_params(acceptable_params,**args):
raise ValueError("At least one parameter with incorrect name given. Check http://devel.yahoo.com/igor/guide/rest_api.html for acceptable parameters")
response = _GET(base_url, method, debug, **args)
# parse the XML response
tree = ElementTree()
tree.parse(response)
pkg_list = []
host_dic = {}
#pkg_dic = {}
set_dic = {}
for elem in tree.iter():
if elem.tag == "state":
host_dic.update(elem.attrib)
if elem.tag == "package":
set_dic = {}
pkg_list.append(elem.attrib)
if elem.tag == "setting":
set_dic.update({ elem.attrib.values()[0] : elem.text })
pkg_list[-1]["setting"] = set_dic
host_dic["packages"] = pkg_list
return Obj(host_dic)
def list_articles(target_directory, supplementary_materials=False, skip=[]):
listing = listdir(target_directory)
for filename in listing:
result_tree = ElementTree()
result_tree.parse(path.join(target_directory, filename))
for tree in result_tree.iterfind('article'):
pmcid = _get_pmcid(tree)
if pmcid in skip:
continue
result = {}
result['name'] = pmcid
result['doi'] = _get_article_doi(tree)
result['article-categories'] = _get_article_categories(tree)
result['article-contrib-authors'] = _get_article_contrib_authors(tree)
result['article-title'] = _get_article_title(tree)
result['article-abstract'] = _get_article_abstract(tree)
result['journal-title'] = _get_journal_title(tree)
result['article-year'], \
result['article-month'], \
result['article-day'] = _get_article_date(tree)
result['article-url'] = _get_article_url(tree)
result['article-license-url'], \
result['article-license-text'], \
result['article-copyright-statement'] = _get_article_licensing(tree)
result['article-copyright-holder'] = _get_article_copyright_holder(tree)
if supplementary_materials:
result['supplementary-materials'] = _get_supplementary_materials(tree)
yield result
def _getPropertyPermission(base_url, method, acceptable_params, debug, **args):
"""Makes the request and parses the response for its wrapper method"""
#check if acceptable criteria was entered (prevents some errors)
if not _correct_params(acceptable_params,**args):
raise ValueError("At least one parameter with incorrect name given. Check http://devel.yahoo.com/igor/guide/rest_api.html for acceptable parameters")
response = _GET(base_url, method, debug, **args)
# parse the XML response
tree = ElementTree()
tree.parse(response)
acl_lst = []
acl_dic = {}
for elem in tree.iter():
if elem.tag == "acl":
if acl_dic:
acl_lst.append( Obj(acl_dic) )
acl_dic = {}
acl_dic.update(elem.attrib)
if elem.tag == "group":
acl_dic[elem.text] = elem.attrib["perm"]
if acl_dic:
acl_lst.append( Obj(acl_dic) )
return acl_lst
def tweak_build_xml(self):
runjdwp_args = [
'transport=dt_socket',
'server=y',
'address=8765',
'suspend=n',
]
runjdwp_args = ','.join(runjdwp_args)
jvm_debug_args = [
'-Xdebug',
'-Xrunjdwp:%s' % (runjdwp_args,),
]
jvm_debug_args = ' '.join(jvm_debug_args)
build_xml = self.get_build_xml()
tree = ElementTree()
tree.parse(build_xml)
root = tree.getroot()
targets = root.findall('target')
for node in targets:
if node.get('name') == 'run':
java_node = node.find('java')
SubElement(java_node, 'jvmarg', {
'line': jvm_debug_args,
})
tree.write(build_xml)
class LastParser(object):
RSS_URL = "http://ws.audioscrobbler.com/2.0/user/{0}/recenttracks.rss"
def __init__(self, user):
self.tree = ElementTree()
self.tree.parse(urllib2.urlopen(self.RSS_URL.format(user)))
def get_songs(self, count=10):
l = []
for item in self.tree.getiterator("item"):
d = {}
for e in item:
d[e.tag] = e.text
l.append(d)
return l[:count]
def get_song(self):
return self.get_songs(1)[0]
def get_titles(self, count=10):
l = [title.text for title in self.tree.getiterator("title")]
return l[1:count + 1] # removing rss title
def get_title(self):
return self.get_titles(1)[0]
def scrape_pkg_uri(self, uri, pkg):
"""
Scrape package metadata from PyPi when it's running
as the Clue Release Manager.
Parameters
----------
uri : `str`
URI to page containing package's homepage
pkg : `str`
Package name
"""
# Example entry:
#<div class="distro-block distro-metadata">
# <h4>Metadata</h4>
# <dl>
# <dt>Distro Index Owner:</dt>
# <dd>acmepypi</dd>
# <dt>Home Page:</dt>
# <dd><a href="http://mysvn/acme.helloworld">
# http://mysvn/acme.helloworld</a></dd>
# </dl>
#</div>
tree = ElementTree()
try:
tree.parse(urllib2.urlopen(uri))
except urllib2.HTTPError, e:
raise UserError("Can't find repository URL for package %s (%s). "
"Has it been registered in PyPi?" % (pkg, e))
def tweak_build_xml(self):
runjdwp_args = [
'transport=dt_socket',
'server=y',
'address=8765',
'suspend=n',
]
runjdwp_args = ','.join(runjdwp_args)
jvm_debug_args = [
'-Xdebug',
'-Xrunjdwp:%s' % (runjdwp_args, ),
]
jvm_debug_args = ' '.join(jvm_debug_args)
build_xml = self.get_build_xml()
tree = ElementTree()
tree.parse(build_xml)
root = tree.getroot()
targets = root.findall('target')
for node in targets:
if node.get('name') == 'run':
java_node = node.find('java')
SubElement(java_node, 'jvmarg', {
'line': jvm_debug_args,
})
tree.write(build_xml)
def scrape_pkg_uri(self, uri, pkg):
"""
Scrape package metadata from PyPi when it's running
as the Clue Release Manager.
Parameters
----------
uri : `str`
URI to page containing package's homepage
pkg : `str`
Package name
"""
# Example entry:
#<div class="distro-block distro-metadata">
# <h4>Metadata</h4>
# <dl>
# <dt>Distro Index Owner:</dt>
# <dd>acmepypi</dd>
# <dt>Home Page:</dt>
# <dd><a href="http://mysvn/acme.helloworld">
# http://mysvn/acme.helloworld</a></dd>
# </dl>
#</div>
tree = ElementTree()
try:
tree.parse(urllib2.urlopen(uri))
except urllib2.HTTPError, e:
raise UserError("Can't find repository URL for package %s (%s). "
"Has it been registered in PyPi?" % (pkg, e))
def mimi_fetch_interactions(gene_id, taxid=None):
gene_id = str(gene_id)
url = MIMI_INT_URL % gene_id
try:
if _SNIPPER_DEBUG:
print "DEBUG: executing MiMI URL %s" % url
except:
pass
xml = urllib2.urlopen(url, timeout=CON_TIMEOUT)
tree = ElementTree()
tree.parse(xml)
go_pattern = re.compile("(.+) \[GO:(\d+)\]")
def extract(pattern, string):
match = pattern.search(string)
if match:
return match.groups()
else:
return (None, None)
results = []
for int_gene in tree.getroot().findall(
"MiMI/Response/ResultSet/Result/InteractingGene"):
other_gene = int_gene.find("GeneID").text
interaction = GeneInteraction(gene_id, other_gene)
for element in int_gene.getchildren():
if element.tag == "TaxonomyID":
interaction.set_tax(element.text)
elif element.tag == "InteractionAttribute":
type = element.get('type')
if type == "Component":
tup = extract(go_pattern, element.text)
interaction.add_component(*tup)
elif type == "Function":
tup = extract(go_pattern, element.text)
interaction.add_function(*tup)
elif type == "Process":
tup = extract(go_pattern, element.text)
interaction.add_process(*tup)
elif type == "Provenance":
interaction.add_provenance(element.text)
elif type == "PubMed":
interaction.add_pubmed(element.text)
elif type == "InteractionType":
interaction.add_interaction_type(element.text)
# Taxonomy ID filter.
if taxid != None:
if interaction.taxon_id != taxid:
continue
results.append(interaction)
return results
def get_major_category_from_pmid(pmid):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml' % pmid
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
for e in tree.iterfind('PubmedArticle/MedlineCitation/MeshHeadingList/MeshHeading/DescriptorName'):
if e.attrib['MajorTopicYN'] == 'Y':
return _postprocess_category(e.text)
def parse_xml(self, xml_file):
tree = ElementTree()
try:
tree.parse(xml_file)
except (SyntaxError):
self.log("parse failed %s" % (xml_file))
return None
return tree
def __init__(self, result_xml):
et = ElementTree()
try:
w = result_xml.encode('utf-8')
def tx(a): return (type(a), len(a))
et.parse(StringIO(w))
except SyntaxError, e:
ptb = traceback.extract_stack()
raise SolrError, (e, result_xml, traceback.format_list(ptb))
def parse_xml(source):
'''Return an ElementTree.
Includes comments and processing instructions.
'''
tree = ElementTree()
pdk_parser = PDKXMLTreeBuilder()
tree.parse(source, pdk_parser)
return tree
def mimi_fetch_interactions(gene_id,taxid=None):
gene_id = str(gene_id);
url = MIMI_INT_URL % gene_id;
try:
if _SNIPPER_DEBUG:
print "DEBUG: executing MiMI URL %s" % url;
except:
pass
xml = urllib2.urlopen(url,timeout=CON_TIMEOUT);
tree = ElementTree();
tree.parse(xml);
go_pattern = re.compile("(.+) \[GO:(\d+)\]");
def extract(pattern,string):
match = pattern.search(string);
if match:
return match.groups();
else:
return (None,None);
results = [];
for int_gene in tree.getroot().findall("MiMI/Response/ResultSet/Result/InteractingGene"):
other_gene = int_gene.find("GeneID").text;
interaction = GeneInteraction(gene_id,other_gene);
for element in int_gene.getchildren():
if element.tag == "TaxonomyID":
interaction.set_tax(element.text);
elif element.tag == "InteractionAttribute":
type = element.get('type');
if type == "Component":
tup = extract(go_pattern,element.text);
interaction.add_component(*tup);
elif type == "Function":
tup = extract(go_pattern,element.text);
interaction.add_function(*tup);
elif type == "Process":
tup = extract(go_pattern,element.text);
interaction.add_process(*tup);
elif type == "Provenance":
interaction.add_provenance(element.text);
elif type == "PubMed":
interaction.add_pubmed(element.text);
elif type == "InteractionType":
interaction.add_interaction_type(element.text);
# Taxonomy ID filter.
if taxid != None:
if interaction.taxon_id != taxid:
continue;
results.append(interaction);
return results;
def set_game_info(self, info):
self.info = info
tree = ElementTree()
tree.parse(info["path"])
link_node = tree.find("link")
if link_node is not None:
self.link_field.set_text(link_node.text.strip())
else:
self.link_field.set_text("")
def __str__(self):
tree = ElementTree()
tree.parse(self.exc)
for elem in tree.iter():
if elem.tag == "msg":
err_msg = elem.text
return "".join(["\n", str(self.exc), "\nAPI Error Message: ", err_msg])
def set_game_info(self, info):
self.info = info
tree = ElementTree()
tree.parse(info["path"])
link_node = tree.find("link")
if link_node is not None:
self.link_field.set_text(link_node.text.strip())
else:
self.link_field.set_text("")
def parse_xml(source):
'''Return an ElementTree.
Includes comments and processing instructions.
'''
tree = ElementTree()
pdk_parser = PDKXMLTreeBuilder()
tree.parse(source, pdk_parser)
return tree
def _get_pmcids_from_dois(dois):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pmc&term=%s' % \
'%20OR%20'.join([doi+'[doi]' for doi in dois])
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
pmcids = []
for e in tree.iterfind('IdList/Id'):
pmcids.append(e.text)
return pmcids
def get_categories_from_pmid(pmid):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml' % pmid
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
categories = []
for e in tree.iterfind('PubmedArticle/MedlineCitation/MeshHeadingList/MeshHeading/DescriptorName'):
category = _postprocess_category(e.text)
categories.append(category)
return categories
def _get_pmcids_from_dois(dois):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pmc&term=%s' % \
'%20OR%20'.join([doi+'[doi]' for doi in dois])
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
pmcids = []
for e in tree.iterfind('IdList/Id'):
pmcids.append(e.text)
return pmcids
def parseStreamListAthanmenu(self):
Athanlist1 = []
tree = ElementTree()
tree.parse(self.xml)
for Athan in tree.findall('Choice'):
Contnt = str(Athan.findtext('Contnt'))
name = str(Athan.findtext('name'))
Athanlist1.append({'Contnt': Contnt, 'name': name})
return Athanlist1
def get_pmid_from_doi(doi):
if not type(doi) == unicode:
raise TypeError, "Cannot get PMID for DOI %s of type %s." % (doi, type(doi))
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=%s' % doi
xml_file = _get_file_from_url(url)
tree = ElementTree()
tree.parse(xml_file)
try:
return int(tree.find('IdList/Id').text)
except AttributeError:
return None
def get_links(self, pkg_name):
"""
Return the simple page links.
"""
tree = ElementTree()
try:
tree.parse(urllib2.urlopen('%s/simple/%s' % (self.uri, pkg_name)))
except (urllib2.HTTPError, exceptions.SyntaxError):
return None
return tree.getroot().findall('body/a')
def get_links(self, pkg_name):
""" Return the simple page links
"""
tree = ElementTree()
try:
tree.parse(urllib2.urlopen('%s/simple/%s' % (self.uri, pkg_name)))
except (urllib2.HTTPError, exceptions.SyntaxError):
return None
ns = self._get_namespace(tree)
return tree.getroot().findall(ns + 'body/ul/li/a')
def parseStreamList(self):
tvlist = []
tree = ElementTree()
tree.parse(self.xml)
for iptv in tree.findall('iptv'):
tvlist.append({'name': str(iptv.findtext('name')).title(),
'icon': str(iptv.findtext('icon')),
'type': str(iptv.findtext('type')),
'uri': self.parseStreamURI(str(iptv.findtext('uri')))})
return sorted(tvlist, key=lambda item: item['name'])
def __init__(self):
self._path = os.path.dirname(os.path.abspath(__file__))
print "Loading Rhythmbox database...",
tree = ElementTree()
tree.parse(database)
self._entries = tree.findall("entry")
self.songs = {}
self.artists = {}
self.albums = {}
self.relationships = {}
self.load_rhythmbox_db()
def parseListAthanAyames(self):
Athanlist1 = []
tree = ElementTree()
tree.parse(self.xml)
for Athan in tree.findall('ayames'):
name = str(Athan.findtext('name'))
date1 = str(Athan.findtext('date1'))
date2 = str(Athan.findtext('date2'))
Athanlist1.append({'name': name, 'date1': date1, 'date2': date2})
return Athanlist1
def parseStreamListAthanmenu_1(self):
Athanlist1 = []
tree = ElementTree()
tree.parse(self.xml)
for Athan in tree.findall('Contry'):
name = str(Athan.findtext('name'))
url = str(Athan.findtext('url'))
Id = str(Athan.findtext('Id'))
Athanlist1.append({'name': name, 'url': url, 'Id': Id})
return Athanlist1
def parseListAthanFlash(self):
Athanlist1 = []
tree = ElementTree()
tree.parse(self.xml)
for Athan in tree.findall('flash'):
name = str(Athan.findtext('name'))
url = str(Athan.findtext('url'))
urimg = str(Athan.findtext('urimg'))
Athanlist1.append({'name': name, 'url': url, 'urimg': urimg})
return Athanlist1
def parseStreamDB(filename):
tvlist = []
tree = ElementTree()
tree.parse(filename)
for iptv in tree.findall('iptv'):
n = str(iptv.findtext('name'))
i = str(iptv.findtext('icon'))
u = str(iptv.findtext('uri'))
t = str(iptv.findtext('type'))
tvlist.append({'name':n, 'icon':i, 'type':t, 'uri':u})
return tvlist
def xml(self, cmd, **kwargs):
# NOTE: darcs is currently encoding agnostic and will print
# patch metadata byte-for-byte, even in the XML changelog.
etree = ElementTree()
# While we are decoding the XML as latin-1 to be as liberal as
# possible, etree will still raise an exception if any
# non-printable characters are in the XML changelog.
parser = XMLParser(encoding='latin-1')
fp = self._run(cmd, **kwargs)
etree.parse(fp, parser=parser)
self.checkexit(fp.close())
return etree.getroot()
def parseStreamList(self):
tvlist = []
tree = ElementTree()
tree.parse(self.xml)
for iptv in tree.findall('iptv'):
n = str(iptv.findtext('name'))
i = str(iptv.findtext('icon'))
u = str(iptv.findtext('uri'))
t = str(iptv.findtext('type'))
tvlist.append({'name':n, 'icon':i, 'type':t, 'uri':self.parseStreamURI(u)})
return tvlist
def _getresponse(self):
r = self.h.getresponse()
if r.status == 200:
et = ElementTree()
et.parse(r)
if et.find('status').text == 'OK':
return et
raise NessusException(et.find('contents').text)
elif r.status == 403:
self.token = None
raise NessusSessionException('Session timed out.')
raise NessusException(r.read())
def fulltext_search(self, query, rows=None, start=None):
"""Does an advanced search on fulltext:blah.
You get back a pair (x,y) where x is the total # of hits
and y is a list of identifiers like ["foo", "bar", etc.]"""
query = self._prefix_query('fulltext', query)
result_list = self.raw_search(query, rows=rows, start=start)
e = ElementTree()
try:
e.parse(StringIO(result_list))
except SyntaxError, e:
raise SolrError, e
def __init__(self, result_xml):
et = ElementTree()
try:
w = result_xml.encode('utf-8')
def tx(a):
return (type(a), len(a))
et.parse(StringIO(w))
except SyntaxError, e:
ptb = traceback.extract_stack()
raise SolrError, (e, result_xml, traceback.format_list(ptb))
def parseStreamDB(filename):
tvlist = []
tree = ElementTree()
tree.parse(filename)
for iptv in tree.findall('iptv'):
n = str(iptv.findtext('name'))
i = str(iptv.findtext('icon'))
u = str(iptv.findtext('uri'))
t = str(iptv.findtext('type'))
tvlist.append({'name':n, 'icon':i, 'type':t, 'uri':u})
return tvlist
def parseStreamList(self):
tvlist = []
tree = ElementTree()
tree.parse(self.xml)
for iptv in tree.findall("iptv"):
tvlist.append({
"name" : str(iptv.findtext("name")).title(),
"icon" : str(iptv.findtext("icon")),
"type" : str(iptv.findtext("type")),
"uri" : self.parseStreamURI(str(iptv.findtext("uri")))
})
return sorted(tvlist, key=lambda item: item["name"])
def updatebq(self):
from xml.etree.cElementTree import ElementTree
tree = ElementTree()
tree.parse(GSXML)
tvlist = []
for iptv in tree.findall("iptv"):
name = iptv.findtext("name").title()
(protocol, serviceType, bufferSize, epgId) = iptv.findtext("type").split(":")
uri = iptv.findtext("uri")
if protocol in "livestreamer":
uri = "http://localhost:88/" + uri
uri = uri.replace(":", "%3a")
service = "#SERVICE {s}:0:1:{e}:{e}:0:0:0:0:0:{u}:{n}\n".format(s=serviceType,e=epgId,u=uri,n=name)
tvlist.append((name,service))
tvlist=sorted(tvlist, key=lambda channel: channel[0]) #sort by name
with open(GSBQ, "w") as f:
f.write("#NAME GreekStreamTV\n")
for (name, service) in tvlist:
f.write(service)
com = "cat /usr/lib/enigma2/python/Plugins/Satdreamgr/UpdateBouquet/stream.xml ; rm /usr/lib/enigma2/python/Plugins/Satdreamgr/UpdateBouquet/stream.xml"
out = os.popen(com)
return list
