def virinSearch(v): vcount=0 countErr=0 loop=True while loop: try: vgen = pagegenerators.SearchPageGenerator(v, namespaces = "6") for vPage in vgen: plist.append(vPage.title()) vcount+=1 loop=False except: loop=True countErr+=1 print Red+"Problem running search, sleeping for",countErr,"seconds",Fore.WHITE time.sleep(countErr) if countErr>30: loop=False vcount=-1 return vcount
def crawlerSearch(pagename): gen = pagegenerators.SearchPageGenerator(pagename, namespaces = "0") for Page in pagegenerators.PreloadingGenerator(gen,10000): modification(Page.title())