Example #1
0
def ingest_edu():
    """
    Pull in categories and feeds from YouTube EDU.
    """
    print "Fetching tags..."
    tags = get_edu_tags()

    # get all courses, using greenlets
    print "Fetching course lists..."
    tag_list = [gevent.spawn(get_courses, tag['term']) for tag in tags]
    gevent.joinall(tag_list)

    # each course_list is a list of courses
    for course_list in tag_list:
        # walk down the list
        # get all lectures for this course list
        courses = [gevent.spawn(get_lectures, c.yt_playlistid) for c in course_list.value]
        gevent.joinall(courses)

        # each lect_list is a list of lectures
        for lect_list in courses:
            # now we can loop through lect_list for videos
            for video in lect_list.value:
                # do something interesting
                store_video(video)

    print "Total videos: %i" % redis.scard(key())
Example #2
0
def update():
    """
    Do the update. Feeds can be for a YouTube user,
    or for a specific feed, like a playlist.
    """
    print "Updating feeds..."
    resp = urllib2.urlopen(FEEDS_URL)
    reader = csv.DictReader(resp)

    for row in reader:
        # get the user or alternate feed
        user = row['User']
        url = row['Feed'] or get_feed_url(user)
                
        # parse whichever we have
        feed = feedparser.parse(url)

        for video in feed.entries:
            store_video(video)

    print "Total videos: %i" % redis.scard(key())