Beispiel #1
0
def ingest_edu():
    """
    Pull in categories and feeds from YouTube EDU.
    """
    print "Fetching tags..."
    tags = get_edu_tags()

    # get all courses, using greenlets
    print "Fetching course lists..."
    tag_list = [gevent.spawn(get_courses, tag['term']) for tag in tags]
    gevent.joinall(tag_list)

    # each course_list is a list of courses
    for course_list in tag_list:
        # walk down the list
        # get all lectures for this course list
        courses = [gevent.spawn(get_lectures, c.yt_playlistid) for c in course_list.value]
        gevent.joinall(courses)

        # each lect_list is a list of lectures
        for lect_list in courses:
            # now we can loop through lect_list for videos
            for video in lect_list.value:
                # do something interesting
                store_video(video)

    print "Total videos: %i" % redis.scard(key())
Beispiel #2
0
def get_random_video():
    """
    Get a random video, pulled from our redis set.

    For stats, increment each video in a sorted set.
    By doing this here, we're checking the randomness
    of videos returned from redis, not the popularity
    of shared videos.
    """
    # get a random video
    uid = redis.srandmember(key())
    user, id = uid.split(':')

    # store stats
    with redis.pipeline() as pipe:
        pipe.zincrby(key('stats'), uid, 1)
        pipe.zincrby(key('stats:users'), user, 1)
        pipe.execute()

    # send us along
    return url_for('video', user=user, id=id)
Beispiel #3
0
def store_video(video):
    """
    Store a video in a redis set.
    """
    # get the video id from the URL
    # there are other ways, but this always works
    vid = get_video_id(video.link)

    # save the username and video id as a key-value pair
    # in our set, so we have both available later
    uid = '%s:%s' % (video.author, vid)

    # save the uid to our redis set
    # printing the video title if it's added to the set
    if redis.sadd(key(), uid):
        log.info(video.title)
Beispiel #4
0
def update():
    """
    Do the update. Feeds can be for a YouTube user,
    or for a specific feed, like a playlist.
    """
    print "Updating feeds..."
    resp = urllib2.urlopen(FEEDS_URL)
    reader = csv.DictReader(resp)

    for row in reader:
        # get the user or alternate feed
        user = row['User']
        url = row['Feed'] or get_feed_url(user)
                
        # parse whichever we have
        feed = feedparser.parse(url)

        for video in feed.entries:
            store_video(video)

    print "Total videos: %i" % redis.scard(key())