def fetch_game(name): print "Looking for: %s" % (name) bin = games_db.find(name) if bin: print " > Found in database." return bin else: try: name = __clean_name(name) gs.search_for_games(name) game_loc = gs.get_next_game() print " > Finding online at: %s" % (game_loc) games_db.add(name, codescraper.fetch_page(game_loc)) return games_db.find(name) except game_searcher.NoResultsException: return None
def _get_more_games(self): #Get google page, and get potential paths query = self._game_query + "&start=" + str(self._next_page) page_text = codescraper.fetch_page(query) #There are 10 results per page, so the next page should be 10 results further along. self._next_page += 10 #This gets all the text between the tags given on the page. url_list = codescraper.return_between("<cite>", "</cite>", page_text) if url_list == []: raise NoResultsException, "No results found!" for a in url_list: #Google sometimes puts html tags in their cite tags like <b> #since these will become messy when you try to create urls from it #we need to remove them. a = codescraper.remove_HTML_tags(a) self._page_list.append(a)