def convert(self, data, cache, **kwargs):
bodydom = Element('div')
kmldom = XML(data)
ns = kmldom.tag.strip('kml')
placemarks = kmldom.findall('.//%sPlacemark' % ns)
for placemark in placemarks:
titles = placemark.findall(ns + 'name')
for title in titles:
t = Element('h2')
t.text = title.text
bodydom.append(t)
descriptions = placemark.findall(ns + 'description')
for desc in descriptions:
if desc.text:
try:
text = desc.text.encode('ascii',
'xmlcharrefreplace').strip()
except:
text = desc.text.strip()
text = sanitize(text)
d = XML('<div>' +
text.encode('ascii', 'xmlcharrefreplace') +
'</div>')
bodydom.append(d)
body = tostring(bodydom)
cache.setData(body)
return cache
def DataFeedQuery(self,start_date,end_date,table_id,report_type,parse_as="flat",sort='',filters='',max_results='50',chart_type="table"):
m_indx = report_type.index("m-")
d_indx = report_type.index("d-")
dimensions = report_type[d_indx+2:m_indx-1]
for d in dimensions.split(","):
d = "ga:" + d +","
dim = d[:-1]
metrics = report_type[m_indx+2:]
for m in metrics.split(","):
m = "ga:" + m +","
metr = m[:-1]
#try:
print "TEST3"
if max_results == '':
max_results = 50
self.data_feed_query = gdata.analytics.client.DataFeedQuery({
'ids': table_id,
'start-date': start_date,
'end-date': end_date,
'dimensions':dim,
'metrics':metr,
'sort': sort,
'filters': filters,
'max-results': max_results})
print self.data_feed_query
self.data_feed = self.my_client.GetDataFeed(self.data_feed_query)
self.parsed_data_feed = XML(str(self.data_feed))
print self.parsed_data_feed
if parse_as == "raw":
return self.parsed_data_feed,len(self.parsed_data_feed.getiterator('{http://www.w3.org/2005/Atom}entry')),True
else:
return self.ParseDataFeedResults(self.parsed_data_feed,parse_as),len(self.parsed_data_feed.getiterator('{http://www.w3.org/2005/Atom}entry')),True
"""except:
def __test_arbitrary_path_rewriting(self):
self.job.parameters = [
JobParameter(name="index_path", value="\"/old/path/human\"")
]
xml = XML('''<param name="index_path" type="select">
<option value="/old/path/human">Human</option>
<option value="/old/path/mouse">Mouse</option>
</param>''')
parameter = SelectToolParameter(self.tool, xml)
def get_field_by_name_for_value(name, value, trans, other_values):
assert value == "/old/path/human"
assert name == "path"
return ["/old/path/human"]
parameter.options = Bunch(
get_field_by_name_for_value=get_field_by_name_for_value)
self.tool.set_params({"index_path": parameter})
self.tool._command_line = "prog1 $index_path.fields.path"
def test_path_rewriter(v):
if v:
v = v.replace("/old", "/new")
return v
self._set_compute_environment(path_rewriter=test_path_rewriter)
command_line, extra_filenames = self.evaluator.build()
self.assertEquals(command_line, "prog1 /new/path/human")
def test_conditional_evaluation(self):
select_xml = XML(
'''<param name="always_true" type="select"><option value="true">True</option></param>'''
)
parameter = SelectToolParameter(self.tool, select_xml)
conditional = Conditional()
conditional.name = "c"
conditional.test_param = parameter
when = ConditionalWhen()
when.inputs = {"thresh": self.tool.test_thresh_param()}
when.value = "true"
conditional.cases = [when]
self.tool.set_params({"c": conditional})
self.job.parameters = [
JobParameter(
name="c",
value=
'''{"thresh": 4, "always_true": "true", "__current_case__": 0}'''
)
]
self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}"
self._set_compute_environment()
command_line, extra_filenames = self.evaluator.build()
self.assertEquals(command_line, "prog1 --thresh=4 --test_param=true")
def create(self, trans, payload, **kwd):
"""
POST /api/forms
Creates a new form.
"""
if not trans.user_is_admin():
trans.response.status = 403
return "You are not authorized to create a new form."
xml_text = payload.get('xml_text', None)
if xml_text is None:
trans.response.status = 400
return "Missing required parameter 'xml_text'."
#enhance to allow creating from more than just xml
form_definition = form_factory.from_elem(XML(xml_text))
trans.sa_session.add(form_definition)
trans.sa_session.flush()
encoded_id = trans.security.encode_id(form_definition.id)
item = form_definition.to_dict(view='element',
value_mapper={
'id':
trans.security.encode_id,
'form_definition_current_id':
trans.security.encode_id
})
item['url'] = url_for('form', id=encoded_id)
return [item]
def test_input_value_wrapper(tool):
parameter = IntegerToolParameter(
tool,
XML('<param name="blah" type="integer" size="4" value="10" min="0" />')
)
wrapper = InputValueWrapper(parameter, "5")
assert str(wrapper) == "5"
def process_rss_feed(request, link_id_encoded, website_encode_id):
from atrinsic.base.models import Link, Website
from atrinsic.web.helpers import base36_decode
from elementtree.ElementTree import XML, tostring
import urllib2
link = Link.objects.get(pk=base36_decode(link_id_encoded))
website = Website.objects.get(pk=base36_decode(website_encode_id))
tracking_url = link.track_html_ape(website, link_only=True)
try:
raw_response = urllib2.urlopen(link.link_content)
except:
return AQ_render_to_response(request,
'base/custom_error.html', {
'errmsg': RSS_TIMEOUT,
},
context_instance=RequestContext(request))
tree = XML(raw_response.read())
for node in tree.getiterator('link'):
domain_position = node.text.find(".com") + 4
node.text = tracking_url + "&url=" + node.text[domain_position:]
return render_to_response("blank_xml_template.html",
{"XML": tostring(tree)},
mimetype="application/xhtml+xml")
def xmltidy(text):
"""send back good lookin' xml (basically a syntax checker)"""
try:
elem = XML(text)
text = tostring(elem)
return text
except:
#print text
return ''
def xml2ListOfDict_Tutorial(xml):
"""Same as previous function, but the hard (wrong) way."""
lis = []
for elt in XML(xml):
dic = {}
for child in elt:
dic[child.tag] = child.text
lis.append(dic)
return lis
def quick_output(format, format_source=None, change_format_xml=None):
test_output = ToolOutput("test_output")
test_output.format = format
test_output.format_source = format_source
if change_format_xml:
test_output.change_format = XML(change_format_xml)
else:
test_output.change_format = None
return test_output
def test_select_wrapper_simple_options(tool):
xml = XML('''<param name="blah" type="select">
<option value="x">I am X</option>
<option value="y" selected="true">I am Y</option>
<option value="z">I am Z</option>
</param>''')
parameter = SelectToolParameter(tool, xml)
wrapper = SelectToolParameterWrapper(parameter, "x", tool.app)
assert str(wrapper) == "x"
assert wrapper.name == "blah"
assert wrapper.value_label == "I am X"
def test_evaluation_of_optional_datasets(self):
# Make sure optional dataset don't cause evaluation to break and
# evaluate in cheetah templates as 'None'.
select_xml = XML(
'''<param name="input1" type="data" optional="true"></param>''')
parameter = DataToolParameter(self.tool, select_xml)
self.job.parameters = [JobParameter(name="input1", value=u'null')]
self.tool.set_params({"input1": parameter})
self.tool._command_line = "prog1 --opt_input='${input1}'"
self._set_compute_environment()
command_line, extra_filenames = self.evaluator.build()
self.assertEquals(command_line, "prog1 --opt_input='None'")
def _setup_blast_tool(tool, multiple=False):
tool.app.write_test_tool_data("blastdb.loc",
"val1\tname1\tpath1\nval2\tname2\tpath2\n")
xml = XML(
'''<param name="database" type="select" label="Nucleotide BLAST database" multiple="%s">
<options from_file="blastdb.loc">
<column name="value" index="0"/>
<column name="name" index="1"/>
<column name="path" index="2"/>
</options>
</param>''' % multiple)
parameter = SelectToolParameter(tool, xml)
return parameter
def param(self):
if not self._param:
multi_text = ""
if self.multiple:
multi_text = 'multiple="True"'
optional_text = ""
if self.optional:
optional_text = 'optional="True"'
template_xml = '''<param name="data2" type="data" ext="txt" %s %s></param>'''
self.param_xml = XML(template_xml % (multi_text, optional_text))
self._param = basic.DataToolParameter(self.mock_tool,
self.param_xml)
return self._param
def _drilldown_parameter(tool):
xml = XML(
'''<param name="some_name" type="drill_down" display="checkbox" hierarchy="recurse" multiple="true">
<options>
<option name="Heading 1" value="heading1">
<option name="Option 1" value="option1"/>
<option name="Option 2" value="option2"/>
<option name="Heading 1" value="heading1">
<option name="Option 3" value="option3"/>
<option name="Option 4" value="option4"/>
</option>
</option>
<option name="Option 5" value="option5"/>
</options>
</param>''')
parameter = DrillDownSelectToolParameter(tool, xml)
return parameter
def param(self):
if not self._param:
multi_text = ""
if self.multiple:
multi_text = 'multiple="True"'
optional_text = ""
if self.optional:
optional_text = 'optional="True"'
options_text = self.options_xml
data_ref_text = ""
if self.set_data_ref:
data_ref_text = 'data_ref="input_bam"'
template_xml = '''<param name="my_name" type="%s" %s %s %s>%s</param>'''
self.param_xml = XML(template_xml %
(self.type, data_ref_text, multi_text,
optional_text, options_text))
self._param = basic.SelectToolParameter(self.mock_tool,
self.param_xml)
return self._param
def param(self):
if not self._param:
multi_text = ""
if self.multiple:
multi_text = 'multiple="True"'
optional_text = ""
if self.optional:
optional_text = 'optional="True"'
data_ref_text = ""
if self.set_data_ref:
data_ref_text = 'data_ref="input_tsv"'
template_xml = '''<param name="my_name" type="%s" %s %s %s %s></param>'''
self.param_xml = XML(template_xml %
(self.type, data_ref_text, multi_text,
optional_text, self.other_attributes))
self._param = basic.ColumnListParameter(self.mock_tool,
self.param_xml)
self._param.ref_input = bunch.Bunch(formats=[
model.datatypes_registry.get_datatype_by_extension("tabular")
])
return self._param
def test_context(self):
if self._test_context is None:
option_xml = ""
if self.filtered_param:
option_xml = '''<options><filter type="data_meta" ref="data1" key="dbkey" /></options>'''
param_xml = XML(
'''<param name="data2" type="data" ext="txt">%s</param>''' %
option_xml)
self.param = basic.DataToolParameter(
tool=self.tool,
elem=param_xml,
)
self._test_context = dataset_matcher.DatasetMatcher(
trans=bunch.Bunch(
app=self.app,
get_current_user_roles=lambda: self.current_user_roles,
workflow_building_mode=True,
),
param=self.param,
value=[],
other_values=self.other_values)
return self._test_context
def parsePodcast(podcastXML, config, filemode=None):
"""
Access to the podcast and return all information in a podcast object
return None when no info available, when error raise MyCancel
"""
podcastInfo = PodcastInfo()
podcastNode = XML(podcastXML)
channelNode = podcastNode.find('channel')
### TODO handle when title is empty, use the program title
titlePodcast = channelNode.findtext('title', '')
titlePodcastAscii = titlePodcast.encode('ascii', 'ignore')
if not titlePodcast:
titlePodcast = 'UNDEFINED'
titlePodcastAscii = 'UNDEFINED'
else:
titlePodcastAscii = getCroppedFilename(titlePodcastAscii)
titlePodcastAscii = cleanString(titlePodcastAscii)
## TODO support the podcast named the same way... add a hashcode after the title, and a main podcast.xml file at the root
# the target local directory
targetDirectory = os.path.join(config.podcastDownloadPath,
titlePodcastAscii)
chandescription = channelNode.findtext('description', '')
#chanImage = getXMLAttrText(channeldom, 'itunes:image', 'href')
####
podcastInfo.title = titlePodcast
podcastInfo.description = chandescription
#podcastInfo.image = chanImage
podcastInfo.titleAscii = titlePodcastAscii
podcastInfo.targetDirectory = targetDirectory
# Get the local info, and a link on the elemnt node, to be modified later
podcastLocalInfo = getPodcastLocalItems(podcastInfo)
# parse the item list
#items = channeldom.getElementsByTagName('item')
itemExist = channelNode.find('item') != None
if not itemExist:
# return empty mark
return None
#for item in items:
for itemNode in channelNode.getiterator('item'):
descr = ''
type = ''
length = ''
title = itemNode.findtext('title', '')
descr = itemNode.findtext('description', '')
#pubDate = getXMLTagText(item, 'pubDate')
#duration = getXMLTagText(item, 'itunes:duration')
enclosureNode = itemNode.find('enclosure')
if enclosureNode == None:
continue
# the url can be redirect, urllib follow this link for downlaod
url = enclosureNode.get('url')
if url == None:
# when no url, continue
continue
# search in the local if the file is already here, already downloaded, and add it
foundLocalItem = None
for podlocalitem in podcastLocalInfo.itemsInfo:
if podlocalitem.url == url:
foundLocalItem = podlocalitem
if foundLocalItem != None:
podcastInfo.itemsInfo.append(foundLocalItem)
continue
# type is not always defined?!, can test with urllib when downloaded
type = enclosureNode.get('type')
# Length only used for information as list, exact size is found later during download
length = enclosureNode.get('length')
#####################
podItem = PodcastItem()
podItem.title = title
podItem.description = descr
podItem.url = url
podItem.type = type
podItem.length = length
podcastInfo.itemsInfo.append(podItem)
# TODO ?? return when ! config.podcastDownload
# init the titles and the filename, filelocation
for podItem in podcastInfo.itemsInfo:
# the display title
title = podItem.title
# when is local, don't process filenames and add >> in the titles
if podItem.isLocal:
# control that the file exist and fully downloaded, when not try to download ...
if not os.path.exists(podItem.fileLocation):
podItem.isLocal = False
podItem.flagfinish = False
elif podItem.flagfinish:
title = '>> ' + title
else:
title = '<> ' + title
# when not local, or not file found
else:
# TODO test that valid name, no special char, len > sufficient, and not always the same name,
# TODO OR add something in the MEDIA xml file
## TODO when a podcast title is many time the same, use a hascode after the title filaname
### PROBELM avec la limite des path: lorsque fichier trop long, peut avec des equivalent......
# podcastInfo.useTitleForName or
if filemode == "title":
filename = podItem.title
podcastInfo.useTitleForName
else:
filename = getLastStringPart(
podItem.url.encode('ascii', 'ignore'), '/')
filename = getBeforeStringPart(filename, '?')
### TODO test if this filename already exit in the list..?? -> if YES MARK it using a poditem flag, and use title...
filename = getCroppedFilename(filename) # 42-4
filename = cleanString(filename)
fileLocation = targetDirectory + '\\' + filename
# set the properties
podItem.filename = filename
podItem.fileLocation = fileLocation
# When the file exist but no entry in the xml add it here
if os.path.exists(podItem.fileLocation):
title = '>? ' + title
# process size
size = 0
if podItem.size != 0:
size = podItem.size
elif podItem.length and len(podItem.length) > 0:
size = round(long(podItem.length) / 1000000.0, 1)
# set size and title
if size != 0:
podItem.size = size
title = title + ' (' + str(size) + 'Mo)'
podcastInfo.itemFilenames.append(podItem.filename)
podcastInfo.titles2display.append(title)
# search in the folder if already downloaded file are available, and not refferenced in the xml
appendLocalMedia(podcastInfo, podcastLocalInfo)
# return the podcast info with items
return podcastInfo
def test_thresh_param(self):
elem = XML('<param name="thresh" type="integer" value="5" />')
return IntegerToolParameter(self, elem)
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title type="xhtml">Atom-Powered <br/> Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>"""
etree = ElementTree(file=StringIO.StringIO(content))
feed = XML(content)
print etree
print feed
#print len(feed)
#print feed[0]
#print feed.keys()
ATOM = "http://www.w3.org/2005/Atom"
entry = etree.getiterator('{%s}entry'%ATOM)[0]
new_lin = SubElement(entry, '{%s}link'%ATOM)
new_lin.set('rel', 'source')
new_lin.set('href', 'http://somthing.org')
typeID = typeID_List[index]
typeID_Progress = 100.0 * ((1.0 * index) / typeID_List_length)
if (verbose == 1):
print "Getting eve-central data for typeID ", typeID_List[
index], " - ", typeName_List[index]
contents = eve_central_query_market_data(regionID, option)
if (verbose == 1):
print "Response from eve-central:"
print contents
print "END RESPONSE"
if (contents[0:5] == "<?xml"):
# Parse the XML response from EVE-Central
# See usage here for ElementTree: http://effbot.org/zone/element-index.htm
tree = XML(contents) # From a string
if (verbose == 1):
print "XML received from eve-central:"
print tree
#print "XML Element count = ", len(tree)
# the tree root is the toplevel html element
tree_map = dict((c, p) for p in tree.getiterator() for c in p)
root = tree
for c in root.getchildren():
if (verbose == 1):
print c.tag
for d in c.getchildren():
if (verbose == 1):
print " ", d.tag
def AccountFeedQuery(self,max_results='50'):
# DataFeedQuery simplifies constructing API queries and uri encodes params.
self.account_feed_query = AccountFeedQuery({'max-results': max_results})
self.account_feed = self.my_client.GetDataFeed(self.account_feed_query)
self.parsed_account_feed = XML(str(self.account_feed))
return self.parsed_account_feed
def _parameter_for(self, **kwds):
content = kwds["xml"]
param_xml = XML( content )
return basic.ToolParameter.build( self.mock_tool, param_xml )
def create(self, trans, payload, **kwd):
"""
POST /api/request_types
Creates a new request type (external_service configuration).
"""
if not trans.user_is_admin():
trans.response.status = 403
return "You are not authorized to create a new request type (external_service configuration)."
xml_text = payload.get('xml_text', None)
if xml_text is None:
trans.response.status = 400
return "Missing required parameter 'xml_text'."
elem = XML(xml_text)
request_form_id = payload.get('request_form_id', None)
if request_form_id is None:
trans.response.status = 400
return "Missing required parameter 'request_form_id'."
request_form = trans.sa_session.query(
trans.app.model.FormDefinition).get(
trans.security.decode_id(request_form_id))
sample_form_id = payload.get('sample_form_id', None)
if sample_form_id is None:
trans.response.status = 400
return "Missing required parameter 'sample_form_id'."
sample_form = trans.sa_session.query(
trans.app.model.FormDefinition).get(
trans.security.decode_id(sample_form_id))
external_service_id = payload.get('external_service_id', None)
if external_service_id is None:
trans.response.status = 400
return "Missing required parameter 'external_service_id'."
external_service = trans.sa_session.query(
trans.app.model.ExternalService).get(
trans.security.decode_id(external_service_id))
request_type = request_type_factory.from_elem(elem, request_form,
sample_form,
external_service)
#FIXME: move permission building/setting to separate abstract method call and
#allow setting individual permissions by role (currently only one action, so not strictly needed)
role_ids = payload.get('role_ids', [])
roles = [
trans.sa_session.query(trans.model.Role).get(
trans.security.decode_id(i)) for i in role_ids
] # if trans.app.security_agent.ok_to_display( trans.user, i ) ]
permissions = {}
if roles:
#yikes, there has to be a better way?
for k, v in trans.model.RequestType.permitted_actions.items():
permissions[trans.app.security_agent.get_action(
v.action)] = roles
if permissions:
trans.app.security_agent.set_request_type_permissions(
request_type, permissions)
#flush objects
trans.sa_session.add(request_type)
trans.sa_session.flush()
encoded_id = trans.security.encode_id(request_type.id)
item = request_type.to_dict(view='element',
value_mapper={
'id': trans.security.encode_id,
'request_form_id':
trans.security.encode_id,
'sample_form_id':
trans.security.encode_id
})
item['url'] = url_for('request_type', id=encoded_id)
return [item]
def xml2ListOfDict(xml):
"""Convert an xmlDoc string to a python list of dictionaries."""
return [
dict([(child.tag, child.text) for child in elt])
for elt in XML(xml)
]
