def scrape_one_user():
"""
Scrape user info about one random user that is not scraped before,
or very long since last scraped
"""
user = db_functions.get_random_user("user")
if not user: return
scrape_user(user)
def scrape_one_user():
"""
Scrape user info about one random user that is not scraped before,
or very long since last scraped
"""
user = db_functions.get_random_user("user")
if not user: return
scrape_user(user)
def scrape_one_tweet():
"""
Scrape tweets of one random user that is not scraped before,
or very long since last scraped
"""
user = db_functions.get_random_user("tweet")
if not user: return
full=False
if user.tweets_last_update_full:
tresh = datetime.datetime.now() - datetime.timedelta(days=twitter_config.TWEET_USER_RESCRAPE_DELTA)
if user.tweets_last_update_full < tresh:
full=True
scrape_tweet(user, full)
def scrape_one_tweet():
"""
Scrape tweets of one random user that is not scraped before,
or very long since last scraped
"""
user = db_functions.get_random_user("tweet")
if not user: return
full = False
if user.tweets_last_update_full:
tresh = datetime.datetime.now() - datetime.timedelta(
days=twitter_config.TWEET_USER_RESCRAPE_DELTA)
if user.tweets_last_update_full < tresh:
full = True
scrape_tweet(user, full)
def scrape_one_friendship():
"""
Scrape all followers and following of one random user that is mot scraped before,
or very long since last scraped
"""
logging.debug(u"Scrape one users friendships")
user = db_functions.get_random_user("friendship")
if not user:
logging.debug(u"user not found")
return
#IF to long since rescraping all friendships, Delete friendships and scrape all from scratch
full=False
if user.friendships_last_update_full:
tresh = datetime.datetime.now() - datetime.timedelta(days=twitter_config.FRIENDSHIP_USER_RESCRAPE_DELTA)
if user.friendships_last_update_full < tresh:
full=True
scrape_friendship(user, full)
def scrape_one_friendship():
"""
Scrape all followers and following of one random user that is mot scraped before,
or very long since last scraped
"""
logging.debug(u"Scrape one users friendships")
user = db_functions.get_random_user("friendship")
if not user:
logging.debug(u"user not found")
return
#IF to long since rescraping all friendships, Delete friendships and scrape all from scratch
full = False
if user.friendships_last_update_full:
tresh = datetime.datetime.now() - datetime.timedelta(
days=twitter_config.FRIENDSHIP_USER_RESCRAPE_DELTA)
if user.friendships_last_update_full < tresh:
full = True
scrape_friendship(user, full)
