def parse_page(browser,soup): list_u = [] p=soup.find(class_='grid_page_center_col results_list') profiles=soup.find_all(class_='pagedlist_item') for profile in profiles: time.sleep(1) l=profile.find('a') link=l['href'] u='https://www.quora.com'+link data=quora.get_quora_data(u,browser) if(data): list_u.append(data) return list_u
def parse_page(browser,dept,soup): list_u = [] p=soup.find(class_="QueryResultsList PagedList") profiles=soup.find_all(class_='pagedlist_item') for profile in profiles: time.sleep(0.8) l=profile.find('a') link=l['href'] u='https://www.quora.com'+link data=quora.get_quora_data(u,dept,browser) if(data): list_u.append(data) return list_u
def parse_page(browser, soup): list_u = [] p = soup.find(class_='grid_page_center_col results_list') profiles = soup.find_all(class_='pagedlist_item') for profile in profiles: time.sleep(1) l = profile.find('a') link = l['href'] u = 'https://www.quora.com' + link data = quora.get_quora_data(u, browser) if (data): list_u.append(data) return list_u
def parse_page(browser, dept, soup): list_u = [] p = soup.find(class_="QueryResultsList PagedList") profiles = soup.find_all(class_='pagedlist_item') for profile in profiles: time.sleep(0.8) l = profile.find('a') link = l['href'] u = 'https://www.quora.com' + link data = quora.get_quora_data(u, dept, browser) if (data): list_u.append(data) return list_u
if login=='y': while 1: soup=bs(browser.page_source) p=soup.find(class_="QueryResultsList PagedList") profiles=soup.find_all(class_='pagedlist_item') for profile in profiles: if(j<i): j+=1 continue j+=1 time.sleep(0.8) l=profile.find('a') link=l['href'] u='https://www.quora.com'+link data={} data=quora.get_quora_data(u,dept,browser2) i=i+1 stdinfo=str(i)+'.json' with open(stdinfo,'w') as f: f.write(data) j=0 elem = browser.find_element_by_tag_name("body") elem.send_keys(Keys.PAGE_DOWN) def parse(url,browser,dept,html,i): #html=browser.page_source soup=bs(html) data={} return parse_page(url,browser,dept,soup,i) import requests