###other random functions def countTags(soup, tag): tags = soup.findAll(tag) return tags.length def printText(tags): for tag in tags: if tag.__class__ == BeautifulSoup.BeautifulSoup.NavigableString: print tag else: printText(tag) if not link_list: print 'no link list' visit_queue, the_url = scrape_bing.get_links(input_keywords, a_list = False) scrape_pages(visit_queue.popleft(), the_url, input_keywords, recurse=True) #search with a suggested link and keywords else: print 'link list' for link in link_list: visit_queue.append(link) first_link = visit_queue.popleft() scrape_pages(first_link, first_link, input_keywords, recurse=True) print visited_pages return list(set(collected_pics)) #print get_collected_pics(['motorcycles'], ['http://www.reddit.com/r/motorcycles', 'http://www.imgur.com/r/motorcycles'])
def scrape_bing_src(keywords): visit_queue, the_url = scrape_bing.get_links(keywords, a_list = False) scrape_pages(visit_queue.popleft(), the_url, keywords, recurse=True)