def performStageOne(data):
soup = bs(openPage(data["sub_channel_link"]))
parsePage(data,soup)
next_link = getCallLink(soup)
updateTrack(data["sub_channel_link"],next_link)
if next_link !="":
performStageTwo(data,next_link)
cursor.execute("UPDATE channel_links SET crawled=1 WHERE sub_channel_link = '%s'"%(data["sub_channel_link"]))
db.commit()
def getLinks(data):
try:
soup = openPage(data["channel_link"])
spans = soup.findAll('span',{'class':'qualified-channel-title-wrapper'})
for span in spans:
try:
link= "https://www.youtube.com"+span.find('a').attrs['href']+"/videos"
name = str(span.text).strip()
storeDatabase((name,link,data["channel_name"]))
except Exception as x:
print (x)
cursor.execute("UPDATE channels SET crawled = 1 WHERE channel_link = '%s' "%(data["channel_link"]))
db.commit()
except Exception as x:
print (x)
