def main(username, password):
try:
browser = Browser('firefox')
except selenium.common.exceptions.WebDriverException:
import os
binpath = os.path.expanduser('~/.local/bin/geckodriver')
tarpath = '/tmp/geckodriver.tar.gz'
os.environ['PATH'] = os.path.dirname(
binpath) + ':' + os.environ['PATH']
if not os.path.exists(binpath):
from six.moves import urllib
from tarfile import TarFile
import shutil
if not os.path.exists(tarpath):
urllib.request.urlretrieve(
'https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux64.tar.gz',
tarpath)
with TarFile.open(tarpath) as f:
f.extractall()
try:
os.makedirs(os.path.dirname(binpath))
except OSError:
pass
shutil.move('geckodriver', binpath)
browser = Browser(
'firefox',
headless=True,
user_agent=
'SoftBank/2.0/004SH/SHJ001/SN 12345678901 Browser/NetFront/3.5 Profile/MIDP-2.0 Configuration/CLDC-1.1'
)
browser.visit('http://furushchev.ru')
if browser.is_element_not_present_by_name('login', wait_time=10):
print('Failed to login or you are already logged in')
else:
browser.fill('SWUserName', username)
browser.fill('SWPassword', password)
browser.find_by_name('login').click()
print('Successfully logged in!')
browser.execute_script(
'document.getElementsByName("formInput")[0].action = "/servlet?p=settings-phonelock&q=write&reboot=false"'
)
browser.execute_script('document.formInput.submit()')
# ===== Change admin password =====
print("\n* Changing admin password")
pass_web_new = "WhichPassWeUse" # NR Should be more secure
# pass_web_new = getpass.getpass(prompt=' Input web admin pass: '******'editOldPassword').first.fill(pass_web_cur)
browser.find_by_name('editNewPassword').first.fill(pass_web_new)
browser.find_by_name('editConfirmPassword').first.fill(pass_web_new)
browser.execute_script('doSubmit()')
browser.visit(yealink_web + "/servlet?p=security&q=load")
if browser.is_element_not_present_by_name('editOldPassword', 1):
print(" Need re-login...")
login_in_web(pass_web_new)
if not username_yealink:
# exit_script(0, True)
break
# ==== Set hostname ====
print("\n* Setting hostname")
browser.visit(yealink_web + "/servlet?p=features-general&q=load")
browser.find_by_name('DhcpHostnameValue').first.fill("SIP-T48G-" +
username_yealink)
browser.execute_script('onNoticeOperator()')
browser.execute_script(
'document.getElementsByName("formInput")[0].action = "/servlet?p=features-general&q=write&reboot=false"'
)
def scrape():
scraped_data = {}
#pointing to the directory where chromedriver exists
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
#visiting the page
url = "https://mars.nasa.gov/news/"
browser.visit(url)
# Wait for article_teaser_body and content_title to load
browser.is_element_not_present_by_id("content_title", wait_time=30)
browser.is_element_not_present_by_id("article_teaser_body", wait_time=30)
#using bs to write it into html
html = browser.html
soup = bs(html, "html.parser")
news_title = soup.find("div", class_="content_title").text
news_paragraph = soup.find("div", class_="article_teaser_body").text
scraped_data['featured_news'] = {
"Title": news_title,
"Paragraph": news_paragraph
}
#Visit the url for JPL Featured Space Image [here](https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars).
#Use splinter to navigate the site and find the image url for the current Featured Mars Image and assign the url string to a
#variable called `featured_image_url`.
#Make sure to find the image url to the full size `.jpg` image.
#Make sure to save a complete url string for this image.
url_image = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
browser.visit(url_image)
#Getting the base url
from urllib.parse import urlsplit
base_url = "{0.scheme}://{0.netloc}/".format(urlsplit(url_image))
#Design an xpath selector to grab the image
xpath = "//*[@id=\"page\"]/section[3]/div/ul/li[1]/a/div/div[2]/img"
#Use splinter to click on the mars featured image
#to bring the full resolution image
# browser.is_element_not_present_by_xpath(xpath, wait_time=30)
results = browser.find_by_xpath(xpath)
img = results[0]
img.click()
browser.is_element_not_present_by_name("fancybox-image", wait_time=30)
#get image url using BeautifulSoup
html_image = browser.html
soup = bs(html_image, "html.parser")
img_url = soup.find("img", class_="fancybox-image")["src"]
full_img_url = base_url + img_url
scraped_data['image_of_the_day'] = {"URL": full_img_url}
#Visit the Mars Weather twitter account [here](https://twitter.com/marswxreport?lang=en) and scrape the latest Mars weather
#tweet from the page. Save the tweet text for the weather report as a variable called `mars_weather`.
url_weather = "https://twitter.com/marswxreport?lang=en"
browser.visit(url_weather)
html_weather = browser.html
soup = bs(html_weather, "html.parser")
#temp = soup.find('div', attrs={"class": "tweet", "data-name": "Mars Weather"})
mars_weather = soup.find(
"p",
class_="TweetTextSize TweetTextSize--normal js-tweet-text tweet-text"
).text
scraped_data["mars_weather"] = {"data": mars_weather}
#Visit the Mars Facts webpage [here](http://space-facts.com/mars/) and use Pandas to scrape the table containing facts about
#the planet including Diameter, Mass, etc.
#Use Pandas to convert the data to a HTML table string.
url_facts = "https://space-facts.com/mars/"
browser.visit(url_facts)
table = pd.read_html(url_facts)
table[0]
df_mars_facts = table[0]
df_mars_facts.columns = ["Parameter", "Values"]
df_mars_facts.set_index(["Parameter"])
mars_html_table = df_mars_facts.to_html()
mars_html_table = mars_html_table.replace("\n", "")
scraped_data['mars_facts_data'] = {"table": mars_html_table}
return scraped_data
