def steperr(row, set_iter, e):
# set_model[set_iter][1] = Status.ERROR #active
set_model[set_iter][7] += 1
Sets.update(error=set_model[set_iter][7]).where(
Sets.id == row['set']).execute()
Urls.update(status=str(e)).where(
Urls.id == row['id']).execute()
def get_comments():
while True:
try:
url = Urls.select().where(Urls.status=='new').limit(1).get().url
# print url
if not url:
break
Urls.update(status='done').where(Urls.url==url).execute()
hotel_id = url.split('/')[2].split('.')[0]
comments_driver.get(base_url + url)
soup = BeautifulSoup(comments_driver.page_source, 'html.parser')
try:
name = soup.find(attrs={'itemprop': 'name'}).text.encode('utf-8')
except Exception, e:
print "hotel name not found ", e
name = hotel_id
Hotel.create(hotel_id=hotel_id, hotel_name=name)
except Exception, e:
print "Insert into hotel error ", e
Urls.update(status='new').where(Urls.url==url).execute()
continue