Python random_wait Examples

Example #1

File: download_table_pages.py Project: jacksingleton/twitter-censorship

def _fetch_page(page):
    logger.info('page=%d from=http', page)
    util.random_wait()
    startat = page * 10
    response = requests.get(config.CHILLING_EFFECTS_URL + str(startat))
    logger.debug('source=%s', response.text)
    return response.text

Example #2

File: find_censored_tweets.py Project: jacksingleton/twitter-censorship

def check_withheld_on_twitter(tweet):
    util.random_wait()
    try:
        tweet_response = requests.get('http://' + tweet).text
        util.random_wait()
        tweet_soup = BeautifulSoup(tweet_response)
        withheld_div = tweet_soup.find('div', class_='tweet-user-withheld')
        return bool(withheld_div)
    except requests.exceptions.ConnectionError:
        return False

Example #3

File: download_notices.py Project: jacksingleton/twitter-censorship

def _fetch_notice(notice_path):
    util.random_wait()
    response = requests.get('https://www.chillingeffects.org' + notice_path)
    return response.text