def process(self, article): """ Freshens the redis entry for `article` """ DataIORedis().connect() key = hmac(article) # Get wiki try: wiki = wikipedia.WikipediaPage(article, preload=True) except DisambiguationError as e: # choose a disambiguation return except PageError as e: # bad page proceed with crawl return # extract & parse html html = parse_strip_elements(wiki.html()) html = parse_convert_links(html) # Get flickr content res = flickr.call('photos_search', {'text': article, 'format': 'json', 'sort': 'relevance', }) # TODO - detect failed responses res_json = json.loads(res[14:-1]) # Extract data for the first photo returned owner = res_json['photos']['photo'][0]['owner'] photo_id = res_json['photos']['photo'][0]['id'] farm = res_json['photos']['photo'][0]['farm'] server = res_json['photos']['photo'][0]['server'] title = res_json['photos']['photo'][0]['title'] secret = res_json['photos']['photo'][0]['secret'] page_content = { 'content': html, 'owner': owner, 'photo_id': photo_id, 'farm': farm, 'server': server, 'title': title, 'secret': secret } DataIORedis().write(key, json.dumps(page_content)) # return the links return wiki.links
def call_flickr(search_str): """Handles calling the flickr API via the local model :param search_str: :return: """ try: res = flickr.call('photos_search', {'text': ' '.join(search_str.split('_')), 'format': 'json', 'sort': 'relevance', 'license': "4,5,7,8" }) except Exception as e: log.error('Flickr api.photos.search failed with: "%s"' % e.message) raise FlickrAPICallError( template='index.html', message='Flickr search request failed "%s"' % search_str, ) return json.loads(res[14:-1])