def ingest_edu():
"""
Pull in categories and feeds from YouTube EDU.
"""
print "Fetching tags..."
tags = get_edu_tags()
# get all courses, using greenlets
print "Fetching course lists..."
tag_list = [gevent.spawn(get_courses, tag['term']) for tag in tags]
gevent.joinall(tag_list)
# each course_list is a list of courses
for course_list in tag_list:
# walk down the list
# get all lectures for this course list
courses = [gevent.spawn(get_lectures, c.yt_playlistid) for c in course_list.value]
gevent.joinall(courses)
# each lect_list is a list of lectures
for lect_list in courses:
# now we can loop through lect_list for videos
for video in lect_list.value:
# do something interesting
store_video(video)
print "Total videos: %i" % redis.scard(key())
def update():
"""
Do the update. Feeds can be for a YouTube user,
or for a specific feed, like a playlist.
"""
print "Updating feeds..."
resp = urllib2.urlopen(FEEDS_URL)
reader = csv.DictReader(resp)
for row in reader:
# get the user or alternate feed
user = row['User']
url = row['Feed'] or get_feed_url(user)
# parse whichever we have
feed = feedparser.parse(url)
for video in feed.entries:
store_video(video)
print "Total videos: %i" % redis.scard(key())
