def web_voyage(self): #Recursive Crawling
# self.update_pendingURLMemory()
while True:
url_current = mx.pickrandom(self.pendingURLMemory)
self.visit(url_current)
time.sleep(0.2)
print('>>> Voyage ENDED')
def web_voyage(self): #Recursive Crawling
while self.pendingURLMemory:
try:
url_current = mx.pickrandom(self.pendingURLMemory)
self.visit(url_current)
except Exception as e:
raise e # | DEPTH 1
print('>>> Voyage ENDED')
def get_random_article():
navdir = 'hyperScraper_profiles/'
randomDomain = navdir + mx.pickrandom(os.listdir(navdir)) + '/data/'
if os.listdir(randomDomain):
randomPage = randomDomain + random.choice(os.listdir(randomDomain))
pagedict = mx.jload(randomPage)
pagedict['localpath'] = randomPage
return pagedict
else:
get_random_article()
def web_voyage(self): #Recursive Crawling
while self.pendingURLMemory:
try:
url_current = mx.pickrandom(self.pendingURLMemory)
self.visit(url_current)
time.sleep(0.2)
except Exception as e:
# pass
raise e
print(e)
print('>>> Voyage ENDED')