def process(self, article):
"""
Freshens the redis entry for `article`
"""
DataIORedis().connect()
key = hmac(article)
# Get wiki
try:
wiki = wikipedia.WikipediaPage(article, preload=True)
except DisambiguationError as e:
# choose a disambiguation
return
except PageError as e:
# bad page proceed with crawl
return
# extract & parse html
html = parse_strip_elements(wiki.html())
html = parse_convert_links(html)
# Get flickr content
res = flickr.call('photos_search', {'text': article,
'format': 'json',
'sort': 'relevance',
})
# TODO - detect failed responses
res_json = json.loads(res[14:-1])
# Extract data for the first photo returned
owner = res_json['photos']['photo'][0]['owner']
photo_id = res_json['photos']['photo'][0]['id']
farm = res_json['photos']['photo'][0]['farm']
server = res_json['photos']['photo'][0]['server']
title = res_json['photos']['photo'][0]['title']
secret = res_json['photos']['photo'][0]['secret']
page_content = {
'content': html,
'owner': owner,
'photo_id': photo_id,
'farm': farm,
'server': server,
'title': title,
'secret': secret
}
DataIORedis().write(key, json.dumps(page_content))
# return the links
return wiki.links
def call_flickr(search_str):
"""Handles calling the flickr API via the local model
:param search_str:
:return:
"""
try:
res = flickr.call('photos_search',
{'text': ' '.join(search_str.split('_')),
'format': 'json',
'sort': 'relevance',
'license': "4,5,7,8"
})
except Exception as e:
log.error('Flickr api.photos.search failed with: "%s"' % e.message)
raise FlickrAPICallError(
template='index.html',
message='Flickr search request failed "%s"' % search_str,
)
return json.loads(res[14:-1])
