def get_new_ip():
while True:
try:
print('GETTING NEW IP')
wrapper.random_connect()
print('SUCCESS')
return
except:
pass
def google_news_run(keyword,
limit=10,
year_start=2010,
year_end=2011,
debug=True,
sleep_time_every_ten_articles=0):
num_articles_index = 0
ua = UserAgent()
while num_articles_index < limit:
url = forge_url(keyword, num_articles_index, year_start, year_end)
if debug:
logging.debug('For Google -> {}'.format(url))
logging.debug('Total number of calls to Google = {}'.format(
NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT))
headers = {'User-Agent': ua.chrome}
success = False
for i in range(20):
try:
response = requests.get(url, headers=headers, timeout=20)
links = extract_links(response.content)
nb_links = len(links)
if nb_links == 0 and num_articles_index == 0:
raise Exception(
'No results fetched. Either the keyword is wrong or you have been banned from Google. Retry tomorrow or change of IP Address.'
)
continue
if nb_links == 0:
print(
'No more news to read for keyword {}.'.format(keyword))
return
outputs = run_parallel_in_threads(get_article, links)
success = True
break
except requests.exceptions.Timeout:
logging.debug(
'Google news TimeOut. Maybe the connection is too slow. Skipping.'
)
continue
if success:
num_articles_index += 10
if debug and sleep_time_every_ten_articles != 0:
logging.debug(
'Program is going to sleep for {} seconds.'.format(
sleep_time_every_ten_articles))
time.sleep(sleep_time_every_ten_articles)
else:
print('Lets change our PUBLIC IP GUYS!')
wrapper.random_connect()
def vpnIpChange():
maxAttempts = 100
attempts = 0
while True:
attempts += 1
try:
logging.info('GETTING NEW IP')
wrapper.random_connect()
log.info('SUCCESS')
return
except Exception as e:
log.error("Exception : {}".format(e))
if attempts > maxAttempts:
log.error('Max attempts reached for VPN. Check its configuration.')
log.error('Browse https://github.com/philipperemy/expressvpn-python.')
log.error('Program will exit.')
exit(1)
def random_vpn(self):
wrapper.random_connect()
return
