def performStageOne(data): soup = bs(openPage(data["sub_channel_link"])) parsePage(data,soup) next_link = getCallLink(soup) updateTrack(data["sub_channel_link"],next_link) if next_link !="": performStageTwo(data,next_link) cursor.execute("UPDATE channel_links SET crawled=1 WHERE sub_channel_link = '%s'"%(data["sub_channel_link"])) db.commit()
def getLinks(data): try: soup = openPage(data["channel_link"]) spans = soup.findAll('span',{'class':'qualified-channel-title-wrapper'}) for span in spans: try: link= "https://www.youtube.com"+span.find('a').attrs['href']+"/videos" name = str(span.text).strip() storeDatabase((name,link,data["channel_name"])) except Exception as x: print (x) cursor.execute("UPDATE channels SET crawled = 1 WHERE channel_link = '%s' "%(data["channel_link"])) db.commit() except Exception as x: print (x)