def get_links():
url = "https://in.mashable.com/"
res = requests.get(url)
data = res.text
soup = BeautifulSoup(data, 'html.parser')
links = set()
for link in soup.findAll(
'a',
attrs={
'href':
re.compile(
"^https://in.mashable.com/tech|https://in.mashable.com/e")
}):
links.add(link.get('href'))
print(highlight_green("\nRetrieved {} links.".format(len(links))))
print(highlight_green("\nArticles fetched:\n\n"))
i = 1
for key in links:
print('{}.'.format(i), end=' ')
print(key)
i += 1
# print('{}.'.format(key+1), end = ' ')
# print(value)
return list(links)
def events(self):
'''This function enables the actions on obstacles and or stoplight
'''
if [15] in self.ids and not self.obstacle_1:
print magenta("Watch out! Unknown obstacle ahead!")
self.obstacle_1 = True
elif self.obstacle_1 and [15] not in self.ids:
self.obstacle_1 = False
if [18] in self.ids:
print magenta(
"Watch out! Dynamic obstacle ahead! Please land and wait for 10 seconds"
)
rospy.sleep(10)
print magenta("You are allowed to start again")
while [14] in self.ids and [13] not in self.ids:
print highlight_red('The light is red! You can not pass.')
rospy.sleep(2)
while [13] in self.ids:
print highlight_green('The light is green! you can proceed.')
rospy.sleep(2)
def identify_leftright(self):
'''
Let user identify which is right and which is left controller by
pulling triggers. Controller hand is displayed in terminal.
'''
while not (rospy.is_shutdown() or self.vive_loc_ready):
rospy.sleep(0.1)
if not rospy.is_shutdown():
print highlight_green(' Pull each trigger ')
identify_right = True
identify_left = True
while (identify_right or identify_left) and not rospy.is_shutdown():
(result, pControllerState) = (self.vrsystem.getControllerState(
self.left_id))
d = self.from_controller_state_to_dict(pControllerState)
# print '\n left controller:', self.last_unPacketNum_left, d['unPacketNum']
# print highlight_green('trigger value: ', d['trigger'])
if (self.last_unPacketNum_left != d['unPacketNum']):
self.last_unPacketNum_left = d['unPacketNum']
# print("Left controller:")
# self.pp.pprint(d)
if d['trigger'] == 1.0:
print highlight_blue(' Left trigger ')
identify_left = False
(result, pControllerState) = (self.vrsystem.getControllerState(
self.right_id))
d = self.from_controller_state_to_dict(pControllerState)
# print 'right controller: ', self.last_unPacketNum_right, d['unPacketNum']
# print highlight_green('trigger value: ', d['trigger'])
if (self.last_unPacketNum_right != d['unPacketNum']):
self.last_unPacketNum_right = d['unPacketNum']
if d['trigger'] == 1.0:
print highlight_blue(' Right trigger ')
identify_right = False
rospy.sleep(1.0 / self.reading_rate_hz)
import pickle
import numpy as np
from fabulous.color import highlight_green
from scraper import get_links, parse_page_data
links = get_links()
data = parse_page_data(links)
model = pickle.load(open("./model/finalized_model.sav", 'rb'))
y = model.predict(data)
indices = np.argwhere(y == np.max(y)).flatten()
print(highlight_green("\nArticles most likely to go viral:\n\n"))
for key, value in enumerate(indices):
print('{}.'.format(key + 1), end=' ')
print(links[value])
#!/usr/bin/env python
# coding=utf-8
from __future__ import print_function
import requests
import os
import sys
from fabulous import text
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
from bs4 import BeautifulSoup
from humanfriendly.tables import format_pretty_table
from fabulous.color import highlight_green, green, red, yellow
baku_header = [
highlight_green('Qatar №-si'.decode("utf-8").strip()),
green('Bakıdan çıxma'.decode("utf-8").strip()),
green('Biləcəriyə çatma'.decode("utf-8").strip()),
yellow('Biləcəridən getmə'.decode("utf-8").strip()),
yellow('Xırdalana çatma'.decode("utf-8").strip()),
red('Xırdalandan getmə'.decode("utf-8").strip()),
red('Sumqayıta çatma'.decode("utf-8").strip())
]
sum_header = [
highlight_green('Qatar №-si'.decode("utf-8").strip()),
green('Sumqayıtdan çıxma'.decode("utf-8").strip()),
green('Xırdalana çatma'.decode("utf-8").strip()),
yellow('Xırdalana getmə'.decode("utf-8").strip()),
yellow('Biləcəriyə çatma'.decode("utf-8").strip()),
red('Biləcəriyə getmə'.decode("utf-8").strip()),
def parse_page_data(links):
page_data = []
encoder = ColumnTransformer(
[('days', OneHotEncoder(categories=[[0, 1, 2, 3, 4, 5, 6]]), [7])],
remainder='passthrough')
for link in links:
res = requests.get(link)
data2 = res.text
soup = BeautifulSoup(data2, 'html.parser')
img_count = 0.0
vid_count = 0.0
for img in soup.findAll('img',
attrs={'itemprop': re.compile("contentUrl")}):
img_count += 1.0
for div in soup.findAll('div', attrs={'class': "vplayer"}):
vid_count += 1.0
# print(vid_count)
post_date = soup.find('time')
weekday = datetime.strptime(post_date['datetime'].split('T')[0],
"%Y-%m-%d").weekday()
# monday = 0 sunday = 6
channel = link.split('/')[3]
title = soup.find('h1', attrs={'id': "id_title"}).text
num_words_title = len(title.strip().split())
num_words_text = len(soup.get_text().strip().split())
if channel == "tech":
data = [
0.0, num_words_title, num_words_text, img_count, vid_count,
0.0, 1.0, weekday
]
elif channel == "entertainment":
data = [
0.0, num_words_title, num_words_text, img_count, vid_count,
1.0, 0.0, weekday
]
else:
data = [
0.0, num_words_title, num_words_text, img_count, vid_count,
0.0, 0.0, weekday
]
if data[-1] == 6 or data[-1] == 5:
data[0] = 1.0
page_data.append(data)
encoded = encoder.fit_transform(page_data)
# print(links[0])
# print(page_data[0])
# print(encoded[0])
print(highlight_green("\nData preprocessing done."))
return encoded
# if __name__ == "main"":
# links = get_links()
# p = parse_page_data(links)
def log(self):
if self.passed:
print highlight_green(self.display_str)
else:
print highlight_red(self.display_str)