def get_youtube_data(video):
"""
Helper to extract video and thumbnail from youtube
"""
video.source = 'youtube'
if 'youtube.com/watch' in video.url:
parsed = urlparse.urlsplit(video.url)
query = urlparse.parse_qs(parsed.query)
try:
video.key = query.get('v')[0]
except IndexError:
video.key = None
else:
video.key = video.url.rsplit('/', 1)[1]
video.embed_src = 'http://www.youtube.com/embed/'
#http://gdata.youtube.com/feeds/api/videos/Agdvt9M3NJA
api_url = 'http://gdata.youtube.com/feeds/api/videos/{}'.format(video.key)
video_data = urlopen(api_url).read()
xml = xmltramp.parse(video_data)
video.title = unicode(xml.title)
video.slug = slugify(video.title)
video.summary = unicode(xml.content)
video.thumb_url = xml[xml_media.group][xml_media.thumbnail:][1]('url')
return video
def serviceGetParameters():
printDebugMessage('serviceGetParameters', 'Begin', 1)
requestUrl = baseUrl + '/parameters'
printDebugMessage('serviceGetParameters', 'requestUrl: ' + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage('serviceGetParameters', 'End', 1)
return doc['id':]
def serviceGetParameters():
printDebugMessage(u"serviceGetParameters", u"Begin", 1)
requestUrl = baseUrl + u"/parameters"
printDebugMessage(u"serviceGetParameters", u"requestUrl: " + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage(u"serviceGetParameters", u"End", 1)
return doc[u"id":]
def getDomainHierarchy():
requestUrl = baseUrl + '/allebi'
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
allebi = doc['domains']['domain']
lower_level_domains = extractLowerLevelDomains(allebi, {})
printDebugMessage('getDomainHierarchy', 'End', 1)
return lower_level_domains
def serviceGetResultTypes(jobId):
printDebugMessage('serviceGetResultTypes', 'Begin', 1)
printDebugMessage('serviceGetResultTypes', 'jobId: ' + jobId, 2)
requestUrl = baseUrl + '/resulttypes/' + jobId
printDebugMessage('serviceGetResultTypes', 'requestUrl: ' + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage('serviceGetResultTypes', 'End', 1)
return doc['type':]
def serviceGetParameterDetails(paramName):
printDebugMessage(u"serviceGetParameterDetails", u"Begin", 1)
printDebugMessage(u"serviceGetParameterDetails", u"paramName: " + paramName, 2)
requestUrl = baseUrl + u"/parameterdetails/" + paramName
printDebugMessage(u"serviceGetParameterDetails", u"requestUrl: " + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage(u"serviceGetParameterDetails", u"End", 1)
return doc
def serviceGetResultTypes(jobId):
printDebugMessage(u"serviceGetResultTypes", u"Begin", 1)
printDebugMessage(u"serviceGetResultTypes", u"jobId: " + jobId, 2)
requestUrl = baseUrl + u"/resulttypes/" + jobId
printDebugMessage(u"serviceGetResultTypes", u"requestUrl: " + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage(u"serviceGetResultTypes", u"End", 1)
return doc[u"type":]
def getDomainsReferencedInEntry(domain, entryid):
printDebugMessage('getDomainsReferencedInEntry', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/entry/' + entryid + '/xref/'
printDebugMessage('getDomainsReferencedInEntry', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
for domain in doc['domains']['domain':]:
print domain('id')
printDebugMessage('getDomainsReferencedInEntry', 'End', 1)
def getDatabaseInfoList():
printDebugMessage('getDatabaseInfoList', 'Begin', 11)
requestUrl = baseUrl + '/dbfetch.databases?style=xml'
xmlDoc = restRequest(requestUrl)
printDebugMessage('getDatabaseInfoList', 'xmlDoc: ' + xmlDoc, 21)
doc = xmltramp.parse(xmlDoc)
databaseInfoList = doc['databaseInfo':]
printDebugMessage('getDatabaseInfoList', 'End', 11)
return databaseInfoList
def getDomainDetails(domain):
printDebugMessage('getDomainDetails', 'Begin', 1)
requestUrl = baseUrl + '/' + domain
printDebugMessage('getDomainDetails', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
domainInfo = doc['domains']['domain']
printDomainDetails(domainInfo)
printDebugMessage('getDomainDetails', 'End', 1)
def getNumberOfResults(domain, query):
printDebugMessage('getNumberOfResults', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '?query=' + query + '&size=0'
printDebugMessage('getNumberOfResults', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
numberOfResults = int(str(doc['hitCount']))
printDebugMessage('getNumberOfResults', 'End', 1)
return numberOfResults
def getMoreLikeThis(domain, entryid, targetDomain, fields, size='', start='', fieldurl='', viewurl='', mltfields='', mintermfreq='', mindocfreq='', maxqueryterm='', excludes='', excludesets=''):
printDebugMessage('getMoreLikeThis', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/entry/' + entryid + '/morelikethis/' + targetDomain +'?size=' +size + '&start=' + start + '&fields=' + fields + '&fieldurl=' + fieldurl + '&viewurl=' + viewurl + '&mltfields=' + mltfields + '&mintermfreq=' + mintermfreq + '&mindocfreq=' + maxqueryterm + '&maxqueryterm=' + mindocfreq + '&excludes=' + excludes + '&excludesets=' + excludesets
printDebugMessage('getMoreLikeThis', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
printEntries(entries)
printDebugMessage('getMoreLikeThis', 'End', 1)
def getResults(domain, query, fields, size='', start='', fieldurl='', viewurl='', sortfield='', order='', sort=''):
printDebugMessage('getResults', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '?query=' + query +'&fields=' + fields + '&size=' + size + '&start=' + start + '&fieldurl=' + fieldurl + '&viewurl=' + viewurl + '&sortfield=' + sortfield + '&order=' + order + '&sort=' + sort
printDebugMessage('getResults', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
printEntries(entries)
printDebugMessage('getResults', 'End', 1)
def serviceGetResultTypes(jobId):
printDebugMessage("serviceGetResultTypes", "Begin", 1)
printDebugMessage("serviceGetResultTypes", "jobId: " + jobId, 2)
requestUrl = baseUrl + "/resulttypes/" + jobId
printDebugMessage("serviceGetResultTypes", "requestUrl: " + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage("serviceGetResultTypes", "End", 1)
return doc["type":]
def serviceGetParameterDetails(paramName):
printDebugMessage('serviceGetParameterDetails', 'Begin', 1)
printDebugMessage('serviceGetParameterDetails', 'paramName: ' + paramName, 2)
requestUrl = baseUrl + '/parameterdetails/' + paramName
printDebugMessage('serviceGetParameterDetails', 'requestUrl: ' + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage('serviceGetParameterDetails', 'End', 1)
return doc
def getAutoComplete(domain, term):
printDebugMessage('getAutoComplete', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/autocomplete?term=' + term
printDebugMessage('getAutoComplete', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
suggestions = doc['suggestions']['suggestion':]
printSuggestions(suggestions)
printDebugMessage('getAutoComplete', 'End', 1)
def getEntries(domain, entryids, fields, fieldurl='', viewurl=''):
printDebugMessage('getEntries', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/entry/' + entryids + '?fields=' + fields + '&fieldurl=' + fieldurl + '&viewurl=' + viewurl
printDebugMessage('getEntries', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
printEntries(entries)
printDebugMessage('getEntries', 'End', 1)
def getTopTerms(domain, field, size='', excludes='', excludesets=''):
printDebugMessage('getTopTerms', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/topterms/' + field + '?size=' + size + '&excludes=' + excludes + '&excludesets=' + excludesets
printDebugMessage('getTopTerms', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
topterms = doc['topTerms']['term':]
for term in topterms:
printTerm(term)
printDebugMessage('getTopTerms', 'End', 1)
def serviceGetParameterDetails(paramName):
printDebugMessage("serviceGetParameterDetails", "Begin", 1)
printDebugMessage("serviceGetParameterDetails", "paramName: " + paramName,
2)
requestUrl = baseUrl + "/parameterdetails/" + paramName
printDebugMessage("serviceGetParameterDetails",
"requestUrl: " + requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
printDebugMessage("serviceGetParameterDetails", "End", 1)
return doc
def getDomainHierarchy():
printDebugMessage('getDomainHierarchy', 'Begin', 1)
requestUrl = baseUrl + '/allebi'
printDebugMessage('getDomainHierarchy', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
allebi = doc['domains']['domain']
printDomains(allebi, '')
domainInfoList = doc['result.domains.domain':]
# for domainInfo in domainInfoList:
# print (domainInfo.id)
printDebugMessage('getDomainHierarchy', 'End', 1)
def getReferencedEntries(domain, entryids, referenceddomain, fields, size='', start='', fieldurl='', viewurl='', facetcount='', facetfields='', facets=''):
printDebugMessage('getReferencedEntries', 'Begin', 1)
requestUrl = baseUrl + '/' + domain + '/entry/' + entryids + '/xref/' + referenceddomain +'?fields=' + fields + '&size=' + size + '&start=' + start + '&fieldurl=' + fieldurl + '&viewurl=' + viewurl + '&facetcount=' + facetcount + '&facetfields=' + facetfields + '&facets=' + facets
printDebugMessage('getReferencedEntries', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
for entry in entries:
printEntries(entry['references']['reference':])
if hasReferenceFacet(entry):
printFacets(entry['referenceFacets']['referenceFacet':])
print
printDebugMessage('getReferencedEntries', 'End', 1)
def getRunLink(run_id):
printDebugMessage('getEntries', 'Begin', 1)
requestUrl = baseUrl + '/metagenomics_runs/entry/' + run_id + '?fieldurl=true'
printDebugMessage('getEntries', requestUrl, 2)
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
fieldURL = ''
for entry in entries:
for fieldurl in entry['fieldURLs']['fieldURL':]:
fieldURL += str(fieldurl)
printDebugMessage('getEntries', 'End', 1)
p = re.compile('http')
fieldURL = p.sub('https', fieldURL)
print fieldURL
def get_vimeo_data(video):
"""
Helper to extract video and thumbnail from vimeo.
"""
#http://vimeo.com/67325705 -- Tumbleweed Tango
video.source = 'vimeo'
video.key = video.url.rsplit('/', 1)[1]
video.embed_src = 'http://player.vimeo.com/video/'
api_url = 'http://vimeo.com/api/v2/video/{}.xml'.format(video.key)
video_data = urlopen(api_url).read()
xml = xmltramp.parse(video_data)
video.title = unicode(xml.video.title)
video.slug = slugify(video.title)
video.summary = unicode(xml.video.description)
video.thumb_url = unicode(xml.video.thumbnail_large)
return video
def get_vimeo_data(video):
"""
Helper to extract video and thumbnail from vimeo.
"""
#http://vimeo.com/67325705 -- Tumbleweed Tango
video.source = 'vimeo'
video.key = video.url.rsplit('/', 1)[1]
video.embed_src = 'http://player.vimeo.com/video/'
api_url = 'http://vimeo.com/api/v2/video/{}.xml'.format(video.key)
video_data = urlopen(api_url).read()
xml = xmltramp.parse(video_data)
video.title = unicode(xml.video.title)
video.slug = slugify(video.title)
video.summary = unicode(xml.video.description)
video.thumb_url = unicode(xml.video.thumbnail_large)
return video
def get_geocode(city, state, street_address="", zipcode=""):
"""
For given location or object, takes address data and returns
latitude and longitude coordinates from Google geocoding service
get_geocode(self, street_address="1709 Grand Ave.", state="MO", zip="64112")
Returns a tuple of (lat, long)
Most times you'll want to join the return.
"""
try:
key = settings.GMAP_KEY
except AttributeError:
return "You need to put GMAP_KEY in settings"
# build valid location string
location = ""
if street_address:
location += '{}+'.format(street_address.replace(" ", "+"))
location += '{}+{}'.format(city.replace(" ", "+"), state)
if zipcode:
location += "+{}".format(zipcode)
url = "http://maps.google.com/maps/geo?q={}&output=xml&key={}".format(
location, key)
file = urllib.urlopen(url).read()
try:
xml = xmltramp.parse(file)
except Exception as error:
print("Failed to parse xml file {}: {}".format(file, error))
return None
status = str(xml.Response.Status.code)
if status == "200":
geocode = str(xml.Response.Placemark.Point.coordinates).split(',')
# Flip geocode because geocoder returns long/lat while Maps wants lat/long.
# Yes, it's dumb.
return (geocode[1], geocode[0])
else:
print(status)
return None
def serviceRun(options):
requestUrl = baseUrl + '/run/'
requestData = urlencode(options)
# Errors are indicated by HTTP status codes.
try:
# Set the HTTP User-agent.
user_agent = getUserAgent()
http_headers = {'User-Agent': user_agent}
req = Request(requestUrl, None, http_headers)
# Make the submission (HTTP POST).
reqH = urlopen(req,
requestData.encode(encoding='utf_8', errors='strict'))
jobId = reqH.read().decode('utf-8')
reqH.close()
except HTTPError as ex:
print(xmltramp.parse(ex.read()))[0][0]
quit()
return jobId
def get_geocode(city, state, street_address="", zipcode=""):
"""
For given location or object, takes address data and returns
latitude and longitude coordinates from Google geocoding service
get_geocode(self, street_address="1709 Grand Ave.", state="MO", zip="64112")
Returns a tuple of (lat, long)
Most times you'll want to join the return.
"""
try:
key = settings.GMAP_KEY
except AttributeError:
return "You need to put GMAP_KEY in settings"
# build valid location string
location = ""
if street_address:
location += '{}+'.format(street_address.replace(" ", "+"))
location += '{}+{}'.format(city.replace(" ", "+"), state)
if zipcode:
location += "+{}".format(zipcode)
url = "http://maps.google.com/maps/geo?q={}&output=xml&key={}".format(location, key)
file = urllib.urlopen(url).read()
try:
xml = xmltramp.parse(file)
except Exception as error:
print("Failed to parse xml file {}: {}".format(file, error))
return None
status = str(xml.Response.Status.code)
if status == "200":
geocode = str(xml.Response.Placemark.Point.coordinates).split(',')
# Flip geocode because geocoder returns long/lat while Maps wants lat/long.
# Yes, it's dumb.
return (geocode[1], geocode[0])
else:
print(status)
return None
def makeRequest(requestUrl):
xmlDoc = restRequest(requestUrl)
doc = xmltramp.parse(xmlDoc)
entries = doc['entries']['entry':]
formatted_output = printEntries(entries)
return formatted_output