def scrape_links(self,pos):
doc_object=MongoDB.get(URLQueue,'document',number=pos)
while doc_object is not None:
self.scrape_link_and_child(doc_object['url'])
pos=MongoDB.increment_url_counter()
doc_object=MongoDB.get(URLQueue,'document',number=pos)
def scrape_urls(cls):
position=MongoDB.get(MetaData,'position',type='queue')
WebScraper().scrape_links(position)
return cls