def populate_friends_from_collection(api,
                                     seed_collection,
                                     friend_collection,
                                     edge_collection=None,
                                     user_sample=1.0,
                                     friends_threshold=20000,
                                     update_threshold=None,
                                     requery=True,
                                     print_progress_every=1000):
    """
    Populates given 'friends_collection' with local user documents representing the friends
    of each user in given 'seed_collection'.
    Note: populated documents are NOT fully hydrated twitter user objects, just IDs with 
    status fields. 
    'friend_ids' field of seed_collection user docs will also be updated with IDs of friends
    fetched from twitter API.
    Parameters:
        api     - fully authenticated Tweepy api or smappPy TweepyPool api
        seed_collection    - fully authenticated (read/write) mongo collection
        friend_collection  - fully authenticated (read/write) mongo collection
        edge_collection    - [OPTIONAL] collection to store simple edge: {to: ID, from: ID}
        user_sample        - proportion of seed users to fetch friends for
        requery            - If False, only query for user's friends if 'friend_ids' field is empty
        friends_threshold  - If user has > friends_threshold, DO NOT QUERY for friends. If user's
                           - doc does not contain 'friends_count', ignore and query anyway
        update_threshold   - Datetime threshold on users to update. Only queries friends of users
                             with 'friends_updated' field LT 'update_threshold'
    """
    # Ensure indexes
    ensure_userdoc_indexes(seed_collection)
    ensure_userdoc_indexes(friend_collection)
    if edge_collection:
        ensure_edge_indexes(edge_collection)

    # Create cursor over users (sample and date restriction possible)
    users = _get_user_sample(seed_collection, user_sample, "friends_updated",
                             update_threshold)

    # Progress vars
    user_count = users.count(with_limit_and_skip=True)
    user_it = 1
    logger.info("Friends: Considering total {0} users".format(user_count))

    # Iterate over users, get friends, save user and friends
    friend_request_failed_for = []
    for user in users:
        # Print progress
        if user_it % print_progress_every == 0:
            print ".. Processing user {0} of {1}".format(user_it, user_count)
        user_it += 1

        # Check user private/deleted fields - don't requery if unreachable
        if "deleted" in user and user["deleted"] == True:
            logging.info("User {0} deleted, skipping".format(user["id"]))
            continue
        elif "private" in user and user["private"] == True:
            logging.info("User {0} private, skipping".format(user["id"]))
            continue

        # Check requery. If false, and user has friend_ids, skip user
        if not requery and user["friend_ids"]:
            logger.debug("User {0} has friends, not re-querying".format(
                user["id"]))
            continue

        # Check friends count for threshold
        if "friends_count" in user and user[
                "friends_count"] > friends_threshold:
            logger.info(
                "User {0} has friends {1} above threshold {2}, skipping".
                format(user["id"], user["friends_count"], friends_threshold))
            continue

        r, friend_ids = get_friends_ids(api, user["id"])
        if _check_return_set_user(r, user, seed_collection):
            logging.info("User {0} unreachable, skipping".format(user["id"]))
            continue

        if friend_ids == None:
            friend_request_failed_for.append(user["id"])
            continue

        # Initialize (if necessary) and set user's friend_ids list
        if not user["friend_ids"]:
            user["friend_ids"] = list(set(friend_ids))
        else:
            user["friend_ids"] = list(set(user["friend_ids"] + friend_ids))

        # Save all friends as userdocs in friends collection
        _save_userdocs(friend_ids, friend_collection)

        # Optionally save "edge" documents
        if edge_collection:
            _save_friend_edges(user["id"], friend_ids, edge_collection)

        # Update user doc's timestamps and save
        user["updated_timestamp"] = datetime.now()
        user["friends_updated"] = datetime.now()
        seed_collection.save(user)

    # Print failure numbers
    logger.info("Failed to find friends for {0} users".format(
        len(friend_request_failed_for)))
def populate_friends_from_collection(api,
                                     seed_collection,
                                     friend_collection,
                                     edge_collection=None,
                                     user_sample=1.0,
                                     friends_threshold=20000,
                                     update_threshold=None,
                                     requery=True,
                                     print_progress_every=1000):
    """
    Populates given 'friends_collection' with local user documents representing the friends
    of each user in given 'seed_collection'.
    Note: populated documents are NOT fully hydrated twitter user objects, just IDs with 
    status fields. 
    'friend_ids' field of seed_collection user docs will also be updated with IDs of friends
    fetched from twitter API.
    Parameters:
        api     - fully authenticated Tweepy api or smappPy TweepyPool api
        seed_collection    - fully authenticated (read/write) mongo collection
        friend_collection  - fully authenticated (read/write) mongo collection
        edge_collection    - [OPTIONAL] collection to store simple edge: {to: ID, from: ID}
        user_sample        - proportion of seed users to fetch friends for
        requery            - If False, only query for user's friends if 'friend_ids' field is empty
        friends_threshold  - If user has > friends_threshold, DO NOT QUERY for friends. If user's
                           - doc does not contain 'friends_count', ignore and query anyway
        update_threshold   - Datetime threshold on users to update. Only queries friends of users
                             with 'friends_updated' field LT 'update_threshold'
    """
    # Ensure indexes
    ensure_userdoc_indexes(seed_collection)
    ensure_userdoc_indexes(friend_collection)
    if edge_collection:
        ensure_edge_indexes(edge_collection)

    # Create cursor over users (sample and date restriction possible)
    users = _get_user_sample(seed_collection, user_sample, "friends_updated",
                             update_threshold)

    # Progress vars
    user_count = users.count(with_limit_and_skip=True)
    user_it = 1
    logger.info("Friends: Considering total {0} users".format(user_count))

    # Iterate over users, get friends, save user and friends
    friend_request_failed_for = []
    for user in users:
        # Print progress
        if user_it % print_progress_every == 0:
            print ".. Processing user {0} of {1}".format(user_it, user_count)
        user_it += 1

        # Check user private/deleted fields - don't requery if unreachable
        if "deleted" in user and user["deleted"] == True:
            logging.info("User {0} deleted, skipping".format(user["id"]))
            continue
        elif "private" in user and user["private"] == True:
            logging.info("User {0} private, skipping".format(user["id"]))
            continue

        # Check requery. If false, and user has friend_ids, skip user
        if not requery and user["friend_ids"]:
            logger.debug("User {0} has friends, not re-querying".format(
                user["id"]))
            continue

        # Check friends count for threshold
        if "friends_count" in user and user[
                "friends_count"] > friends_threshold:
            logger.info(
                "User {0} has friends {1} above threshold {2}, skipping".
                format(user["id"], user["friends_count"], friends_threshold))
            continue

        r, friend_ids = get_friends_ids(api, user["id"])
        if _check_return_set_user(r, user, seed_collection):
            logging.info("User {0} unreachable, skipping".format(user["id"]))
            continue

        if friend_ids == None:
            friend_request_failed_for.append(user["id"])
            continue

        # Initialize (if necessary) and set user's friend_ids list
        if not user["friend_ids"]:
            user["friend_ids"] = list(set(friend_ids))
        else:
            user["friend_ids"] = list(set(user["friend_ids"] + friend_ids))

        # Save all friends as userdocs in friends collection
        _save_userdocs(friend_ids, friend_collection)

        # Optionally save "edge" documents
        if edge_collection:
            _save_friend_edges(user["id"], friend_ids, edge_collection)

        # Update user doc's timestamps and save
        user["updated_timestamp"] = datetime.now()
        user["friends_updated"] = datetime.now()
        seed_collection.save(user)

    # Print failure numbers
    logger.info("Failed to find friends for {0} users".format(
        len(friend_request_failed_for)))
def populate_followers_from_collection(api,
                                       seed_collection,
                                       follower_collection,
                                       edge_collection=None,
                                       user_sample=1.0,
                                       followers_threshold=20000,
                                       update_threshold=None,
                                       requery=True,
                                       print_progress_every=1000):
    """
    See 'populate_friends_from_collection'. Exactly the same, but for followers
    """
    # Ensure indexes
    ensure_userdoc_indexes(seed_collection)
    ensure_userdoc_indexes(follower_collection)
    if edge_collection:
        ensure_edge_indexes(edge_collection)

    # Create cursor over users (sample and date restriction possible)
    users = _get_user_sample(seed_collection, user_sample, "followers_updated",
                             update_threshold)

    # Progress vars
    user_count = users.count(with_limit_and_skip=True)
    user_it = 1
    logger.info("Considering total {0} users".format(user_count))

    # Iterate over users, get followers, save user and followers
    follower_request_failed_for = []
    for user in users:
        if user_it % print_progress_every == 0:
            print ".. Processing user {0} of {1}".format(user_it, user_count)
        user_it += 1

        # Check user private/deleted fields - don't requery if unreachable
        if "deleted" in user and user["deleted"] == True:
            logging.info("User {0} deleted, skipping".format(user["id"]))
            continue
        elif "private" in user and user["private"] == True:
            logging.info("User {0} private, skipping".format(user["id"]))
            continue

        # Check requery. If false, and user has follower_ids, skip user
        if not requery and user["follower_ids"]:
            logger.debug("User {0} has followers, not re-querying".format(
                user["id"]))
            continue

        # Check followers count for threshold
        if "followers_count" in user and user[
                "followers_count"] > followers_threshold:
            logger.info(
                "User {0} has followers {1} above threshold {2}, skipping".
                format(user["id"], user["followers_count"],
                       followers_threshold))
            continue

        r, follower_ids = get_followers_ids(api, user["id"])
        if _check_return_set_user(r, user, seed_collection):
            logging.info("User {0} unreachable, skipping".format(user["id"]))
            continue

        if follower_ids == None:
            follower_request_failed_for.append(user["id"])
            continue

        # Initialize (if necessary) and set user's follower_ids list
        if not user["follower_ids"]:
            user["follower_ids"] = list(set(follower_ids))
        else:
            user["follower_ids"] = list(
                set(user["follower_ids"] + follower_ids))

        # Save all followers as userdocs in followers collection
        _save_userdocs(follower_ids, follower_collection)

        # Optionally save "edge" documents
        if edge_collection:
            _save_follower_edges(user["id"], follower_ids, edge_collection)

        # Update user doc's timestamps and save
        user["updated_timestamp"] = datetime.now()
        user["followers_updated"] = datetime.now()
        seed_collection.save(user)

    # Print failure numbers
    logger.info("Failed to find followers for {0} users".format(
        len(follower_request_failed_for)))
def populate_followers_from_collection(api,
                                       seed_collection,
                                       follower_collection,
                                       edge_collection=None,
                                       user_sample=1.0,
                                       followers_threshold=20000,
                                       update_threshold=None,
                                       requery=True,
                                       print_progress_every=1000):
    """
    See 'populate_friends_from_collection'. Exactly the same, but for followers
    """
    # Ensure indexes
    ensure_userdoc_indexes(seed_collection)
    ensure_userdoc_indexes(follower_collection)
    if edge_collection:
        ensure_edge_indexes(edge_collection)

    # Create cursor over users (sample and date restriction possible)
    users = _get_user_sample(seed_collection, user_sample, "followers_updated",
                             update_threshold)

    # Progress vars
    user_count = users.count(with_limit_and_skip=True)
    user_it = 1
    logger.info("Considering total {0} users".format(user_count))

    # Iterate over users, get followers, save user and followers
    follower_request_failed_for = []
    for user in users:
        if user_it % print_progress_every == 0:
            print ".. Processing user {0} of {1}".format(user_it, user_count)
        user_it += 1

        # Check user private/deleted fields - don't requery if unreachable
        if "deleted" in user and user["deleted"] == True:
            logging.info("User {0} deleted, skipping".format(user["id"]))
            continue
        elif "private" in user and user["private"] == True:
            logging.info("User {0} private, skipping".format(user["id"]))
            continue

        # Check requery. If false, and user has follower_ids, skip user
        if not requery and user["follower_ids"]:
            logger.debug("User {0} has followers, not re-querying".format(
                user["id"]))
            continue

        # Check followers count for threshold
        if "followers_count" in user and user[
                "followers_count"] > followers_threshold:
            logger.info(
                "User {0} has followers {1} above threshold {2}, skipping".
                format(user["id"], user["followers_count"],
                       followers_threshold))
            continue

        r, follower_ids = get_followers_ids(api, user["id"])
        if _check_return_set_user(r, user, seed_collection):
            logging.info("User {0} unreachable, skipping".format(user["id"]))
            continue

        if follower_ids == None:
            follower_request_failed_for.append(user["id"])
            continue

        # Initialize (if necessary) and set user's follower_ids list
        if not user["follower_ids"]:
            user["follower_ids"] = list(set(follower_ids))
        else:
            user["follower_ids"] = list(
                set(user["follower_ids"] + follower_ids))

        # Save all followers as userdocs in followers collection
        _save_userdocs(follower_ids, follower_collection)

        # Optionally save "edge" documents
        if edge_collection:
            _save_follower_edges(user["id"], follower_ids, edge_collection)

        # Update user doc's timestamps and save
        user["updated_timestamp"] = datetime.now()
        user["followers_updated"] = datetime.now()
        seed_collection.save(user)

    # Print failure numbers
    logger.info("Failed to find followers for {0} users".format(
        len(follower_request_failed_for)))