def startMobileDriver(self):
options = Options()
options.set_headless(self.useHeadless)
if self.loadDefaultProfile == True:
firefoxMobileProfile = webdriver.FirefoxProfile(profile_directory=self.ffProfileDir)
else:
firefoxMobileProfile = None
if self.mobileUA != None:
firefoxMobileProfile.set_preference("general.useragent.override", self.mobileUA)
self.firefoxMobileDriver = webdriver.Firefox(
firefox_profile=firefoxMobileProfile, executable_path=self.driverBinary, firefox_options=options)
self.mobileRunning = True
if self.loadCookies == True and self.cookies != None:
self.firefoxMobileDriver.delete_all_cookies()
# Loading cookies only works if we are at a site the cookie would work for
self.getMobileUrl("https://login.live.com")
for cookie in self.cookies:
# print("Adding cookie to Firefox Mobile Driver: %s" % str(cookie))
# new_cookie = {}
# new_cookie['name'] = cookie['name']
# new_cookie['value'] = cookie['value']
try:
self.firefoxMobileDriver.add_cookie(cookie)
except selenium.common.exceptions.InvalidCookieDomainException:
contine
def redeem(codes, headless=True):
try:
opts = Options()
if headless:
opts.set_headless()
ffprofile = webdriver.FirefoxProfile(settings.firefox_profile_path)
browser = webdriver.Firefox(firefox_profile=ffprofile, options=opts)
browser.get('http://www.marvel.com/redeem')
redeem_code(browser, codes)
browser.quit()
except:
logging.error('Unable to redeem: %s' % ','.join(codes))
def EventsList(events_url):
options = Options() #Grabs the first 4 events from the events page
options.set_headless(headless=True)
browser = webdriver.Firefox(firefox_options=options)
try:
browser.get(events_url)
innerHTML = browser.execute_script("return document.body.innerHTML")
test = (browser.page_source)
browser.quit()
except:
test = "null"
try:
text = test.split('hovercard/event.php?id=')[1]
text1 = text.split('hovercard/event.php?id=')[0]
text1 = text1.split('"')[0]
latest = text1
except:
logs.print2("Couldn't extract latest event id")
latest = "null"
try:
text = test.split('hovercard/event.php?id='+text1)[1]
text2 = text.split('hovercard/event.php?id=')[1]
text2 = text2.split('hovercard/event.php?id=')[0]
text2 = text2.split('"')[0]
first = text2
except:
logs.print2("Couldn't extract first event id")
first = "null"
try:
text = test.split('hovercard/event.php?id='+text2)[1]
text3 = text.split('hovercard/event.php?id=')[1]
text3 = text3.split('hovercard/event.php?id=')[0]
text3 = text3.split('"')[0]
second = text3
except:
logs.print2("Couldn't extract second event id")
second = "null"
try:
text = test.split('hovercard/event.php?id='+text3)[1]
text4 = text.split('hovercard/event.php?id=')[1]
text4 = text4.split('hovercard/event.php?id=')[0]
text4 = text4.split('"')[0]
third = text4
except:
logs.print2("Couldn't extract third event id")
third = "null"
return (latest,first,second,third)
def _get_Firefox(self):
try:
bin_path = self.config.get('firefox_binary_path')
binary = FirefoxBinary(bin_path)
geckodriver_path = self.config.get('geckodriver_path')
options = FirefoxOptions()
profile = webdriver.FirefoxProfile()
options.add_argument(
'user-agent={}'.format(self.user_agent))
if self.browser_mode == 'headless':
options.set_headless(headless=True)
#options.add_argument('window-size=1200x600') # optional
if self.proxy:
# this means that the proxy is user set, regardless of the type
profile.set_preference("network.proxy.type", 1)
if self.proxy.proto.lower().startswith('socks'):
profile.set_preference("network.proxy.socks", self.proxy.host)
profile.set_preference("network.proxy.socks_port", self.proxy.port)
profile.set_preference("network.proxy.socks_version", 5 if self.proxy.proto[-1] == '5' else 4)
profile.update_preferences()
elif self.proxy.proto == 'http':
profile.set_preference("network.proxy.http", self.proxy.host)
profile.set_preference("network.proxy.http_port", self.proxy.port)
else:
raise ValueError('Invalid protocol given in proxyfile.')
profile.update_preferences()
self.webdriver = webdriver.Firefox(firefox_binary=binary, firefox_options=options,
executable_path=geckodriver_path, firefox_profile=profile)
return True
except WebDriverException as e:
# reaching here is bad, since we have no available webdriver instance.
logger.error(e)
return False
outputdir = '/home/firefox/output/'
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
profile = webdriver.FirefoxProfile()
# USERAGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
# profile.set_preference("general.useragent.override", USERAGENT)
# profile.set_preference("permissions.default.image", 2)
# profile.set_preference('permissions.default.stylesheet', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
profile.set_preference("network.proxy.ftp", "localhost")
profile.set_preference("network.proxy.ftp_port", 8080)
profile.set_preference("network.proxy.http", "localhost")
profile.set_preference("network.proxy.http_port", 8080)
profile.set_preference("network.proxy.socks", "localhost")
profile.set_preference("network.proxy.socks_port", 8080)
profile.set_preference("network.proxy.ssl", "localhost")
profile.set_preference("network.proxy.ssl_port", 8080)
profile.set_preference("network.proxy.type", 1)
options = Options()
options.set_headless(headless=True)
driver = webdriver.Firefox(executable_path='/usr/bin/geckodriver', options=options, firefox_profile=profile)
try:
driver.get(url)
sleep(3)
print('title:', driver.title)
driver.save_screenshot(outputdir + 'screenshot-' + timestamp + '.png')
finally:
driver.quit()
def __init__(self, client="firefox", username="******", proxy=None, command_executor=None, loadstyles=False,
profile=None, headless=False, autoconnect=True, logger=None, extra_params=None, chrome_options=None):
"""Initialises the webdriver"""
self.logger = logger or self.logger
extra_params = extra_params or {}
if profile is not None:
self._profile_path = profile
self.logger.info("Checking for profile at %s" % self._profile_path)
if not os.path.exists(self._profile_path):
self.logger.critical("Could not find profile at %s" % profile)
raise WhatsAPIException("Could not find profile at %s" % profile)
else:
self._profile_path = None
self.client = client.lower()
if self.client == "firefox":
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
if not loadstyles:
# Disable CSS
self._profile.set_preference('permissions.default.stylesheet', 2)
# Disable images
self._profile.set_preference('permissions.default.image', 2)
# Disable Flash
self._profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
if proxy is not None:
self.set_proxy(proxy)
options = Options()
if headless:
options.set_headless()
options.profile = self._profile
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities['webStorageEnabled'] = True
self.logger.info("Starting webdriver")
self.driver = webdriver.Firefox(capabilities=capabilities, options=options, **extra_params)
elif self.client == "chrome":
self._profile = webdriver.ChromeOptions()
if self._profile_path is not None:
self._profile.add_argument("user-data-dir=%s" % self._profile_path)
if proxy is not None:
self._profile.add_argument('--proxy-server=%s' % proxy)
if headless:
self._profile.add_argument('headless')
if chrome_options is not None:
for option in chrome_options:
self._profile.add_argument(option)
self.logger.info("Starting webdriver")
self.driver = webdriver.Chrome(options=self._profile, **extra_params)
elif client == 'remote':
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
capabilities = DesiredCapabilities.FIREFOX.copy()
self.driver = webdriver.Remote(
command_executor=command_executor,
desired_capabilities=capabilities,
**extra_params
)
else:
self.logger.error("Invalid client: %s" % client)
self.username = username
self.wapi_functions = WapiJsWrapper(self.driver, self)
self.driver.set_script_timeout(500)
self.driver.implicitly_wait(10)
if autoconnect:
self.connect()
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
opts = Options()
opts.set_headless()
assert opts.headless
browser = webdriver.Firefox(options=opts)
browser.get('https://duckduckgo.com')
search_form = browser.find_element_by_id('search_form_input_homepage')
search_form.send_keys('real python')
search_form.submit()
results = browser.find_elements_by_class_name("result__a")
print(results)
browser.close()
quit()
def setUp(self):
options = Options()
options.set_headless(headless=True)
self.browser = webdriver.Firefox(firefox_options=options)
self.browser.implicitly_wait(1)
# get a user and make sure they are logged in for each test
self.user = User.objects.get(username='******')
self.user.profile.account_type = 'artist'
self.user.save()
client = Client()
client.force_login(self.user)
self.browser.get(self.live_server_url)
self.browser.add_cookie({
'name': 'sessionid',
'value': client.cookies.get('sessionid').value
})
def tearDown(self):
self.browser.quit()
#User with an artist goes to a venue detail page
#User clicks requeast to play at venue
#A requestgigVenue is created with to, from, time and gig_description
#request is default unconfirmed
#request shows up in artists dashboard
#requset can be confirmed or denied by venue owner
def test_venue_request_create(self):
self.browser.get(
self.live_server_url +
reverse('venue_detail', args=('lT75sSwrUWcVieIYIaQs', )))
self.browser.find_element_by_id('request_gig_at_venue').click()
self.assertIn('Request', self.browser.title)
self.browser.find_element_by_id('id_gig_request_name').send_keys(
'TestReq5')
self.browser.find_element_by_id(
'id_gig_request_description').send_keys('a description')
self.browser.find_element_by_id('id_gig_request_date').send_keys(
'01/01/2020')
self.browser.find_element_by_id('submit').click()
self.assertIn('created successfully',
self.browser.find_element_by_id('messages').text)
self.assertIn(
'TestReq5',
self.browser.find_element_by_id('link_TestReq5').text)
def test_venue_request_read(self):
# addfixture to create request
self.browser.get(self.live_server_url +
reverse('artist_dashboard'))
self.assertIn(
'James pub',
self.browser.find_element_by_id('link_GigReqTest3').text)
self.browser.find_element_by_id('link_GigReqTest3').click()
self.assertIn(
'James pub',
self.browser.find_element_by_id('gig_request_venue').text)
def test_venue_request_update(self):
# addfixture to create request
self.browser.get(self.live_server_url +
reverse('artist_dashboard'))
self.assertIn(
'GigReqTest3',
self.browser.find_element_by_id('link_GigReqTest3').text)
self.browser.find_element_by_id('link_GigReqTest3').click()
gig_id = self.browser.find_element_by_id('gig_request_id').text
self.browser.find_element_by_id('update_gig_request').click()
self.browser.find_element_by_id(
'id_gig_request_description').clear()
self.browser.find_element_by_id(
'id_gig_request_description').send_keys('a new description')
self.browser.find_element_by_id('submit').click()
self.browser.find_element_by_id('link_GigReqTest3').click()
self.assertIn(
'a new description',
self.browser.find_element_by_id(
'gig_request_description').text)
def test_venue_request_delete(self):
self.browser.get(self.live_server_url +
reverse('artist_dashboard'))
self.assertIn('Jim',
self.browser.find_element_by_id('artist_name').text)
self.assertIn(
'James pub',
self.browser.find_element_by_id('link_GigReqTest3').text)
self.browser.find_element_by_id('link_GigReqTest3').click()
self.browser.find_element_by_id('delete_request').click()
self.browser.find_element_by_id('submit').click()
self.assertIn('deleted',
self.browser.find_element_by_id('messages').text)
#logic to see if object delteted
def test_venue_request_respond(self):
# addfixture to create request
self.browser.get(self.live_server_url +
reverse('artist_dashboard'))
self.assertIn(
'James pub',
self.browser.find_element_by_id('link_GigReqTest3').text)
self.browser.find_element_by_id('link_GigReqTest3').click()
self.browser.find_element_by_id('submit').click()
self.browser.get(self.live_server_url + reverse('my_gigs'))
self.assertIn(
'James pub',
self.browser.find_element_by_id('link_GigReqTest3').text)
translistvals = []
for val in kvdict.values():
translistvals.append(val)
newvals = []
for item in translistvals:
if '\\n' in item:
item = item.split('\\n')
newvals.append(item)
sepdict = {}
for (k, v) in zip(translistkeys, newvals):
sepdict[k] = v
options = Options()
options.set_headless()
firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
False)
assert options.headless
driver = webdriver.Firefox(firefox_options=options,
firefox_profile=firefox_profile)
driver.get("https://translate.google.co.uk/#en/" + langcode + "/")
newrefvals = []
itemlist = list(sepdict.items())
for key, value in sepdict.items():
x = False
if (type(value) is list):
tempvals = []
# from selenium.webdriver.support.ui import WebDriverWait
# from selenium.webdriver.support import expected_conditions as EC
import time
#4) stadium name
# https://en.wikipedia.org/wiki/List_of_national_stadiums
# stadium_image_url
# https://en.wikipedia.org/wiki/Estadio_Monumental_Antonio_Vespucio_Liberti
CITY = 3
STADIUM_NAME = 4 #CLICKABLE
CAPACITY = 5
BUILT_DATE = 6
opts = Options()
opts.set_headless = True
driver = webdriver.Firefox(options=opts)
driver.get('http://www.worldstadiums.com/')
time.sleep(3)
#click on the continent
continent_buttons = driver.find_elements_by_xpath("//td[@bgcolor='#e6e6ff']/a")
continent_buttons[3].click()
time.sleep(5)
#driver_1 = driver_1.page_source
#latest_window = driver.window_handles[1]
#driver.switch_to(window(latest_window))
#source_code = driver.page_source
def setUpClass(cls):
super().setUpClass()
opts = Options()
opts.set_headless()
cls.browser = webdriver.Firefox(options=opts)
cls.browser.set_window_size(1024, 768)
def setUp(self):
options = Options()
options.set_headless(headless=True)
self.browser = webdriver.Firefox(firefox_options=options)
self.browser.implicitly_wait(1)
self.user = User.objects.get(username='******')
def initialize():
global selenium
if not selenium:
opts = Options()
opts.set_headless()
selenium = Firefox(options=opts)
def __init__(self,start_url=START_URL):
opts = Options() # call optoins for Firefox browser
opts.set_headless()
assert opts.headless # set hidden mode for browser
self.browser = webdriver.Firefox(options=opts) # run browser
self.browser.get(start_url) # open start_url
def getCollections():
#counter for tallying collections
counter = 0
#list to store all the JSON data of collections.
collections = []
#options for headless mode
options = Options()
options.set_headless(headless=True)
#open browser
driver = selenium.webdriver.Firefox(firefox_options=options,
executable_path=r'./geckodriver')
#make url varaible to hold url
url = "https://www.nasa.gov/multimedia/imagegallery/iotd.html"
#load url
driver.get(url)
#Yamini Maddelawait until the button loads to load more images
#try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "trending")))
#once the button loads, click it 16 times
# i =0
# while i <= 15:
# driver.find_element_by_xpath('//*[@id="trending"]').click()
# i += 1
#slow down the load to insure button loads correctly
# time.sleep(1)
#click on the first card or picture
driver.find_element_by_class_name('gallery-card:first-child').click()
i = 0
while i <= 3:
#get the hreh link for current page
date_url = driver.find_element_by_partial_link_text('View Image')
#collection object
collection = {}
collection["Creator"] = "Nasa image of the Day"
collection["URL"] = "https://nasa.gov"
collection["ReferenceURL"] = url
#new dictionary for contact information
contact_JSON = {}
contact_JSON["Name"] = "Robert Nemiroff"
contact_JSON["Email"] = "*****@*****.**"
contact_JSON["Telephone"] = "1-906-487-2198"
contact_JSON["Address"] = "1400 Townsend Drive"
contact_JSON["City"] = "Houghton"
contact_JSON["StateProvince"] = "Michigan"
contact_JSON["PostalCode"] = "49931"
contact_JSON["Country"] = "USA"
#add contact_JSON object to collection Object
collection["Contact"] = contact_JSON
#get the description
description = driver.find_element_by_tag_name('p')
collection["Description"] = description.text
#get the rest of the image collection
get_json(date_url.get_attribute("href"), collection)
#add current collection JSON object to collections list
collections.append(collection)
#after collectiong, click on the next button for next picture
driver.find_element_by_class_name('flex-next').click()
i += 1
#close the driver
driver.close()
return {"collections": collections, "count": counter}
def get_page_with_selenium(query, w=True):
# COMMENT
"""
geckodriver&firefox: https://elements.heroku.com/buildpacks/evosystem-jp/heroku-buildpack-firefox
add RedosToGo on Heroku
:param query:
:param w:
:return:
"""
yandex_dict = {}
opts = Options()
opts.set_headless()
# binary = FirefoxBinary('/app/vendor/firefox/firefox')
# driver = Firefox(options=opts, firefox_binary=binary, executable_path='/app/vendor/geckodriver/geckodriver')
# driver = Firefox(options=opts, firefox_binary=binary)
driver = Firefox(options=opts)
driver.wait = WebDriverWait(driver, 5)
driver.get('https://www.yandex.ru')
html = driver.page_source
if w:
print(1)
with open('index.html', 'w') as f:
f.write(html)
# try:
# #
# # # box = driver.find_element_by_id('text')
# #
# box = driver.wait.until(EC.presence_of_element_located(
# (By.ID, 'text'),
# ))
# # print(12)
# # button = driver.find_element_by_class_name('suggest2-form__button')
# button = driver.wait.until(EC.presence_of_element_located(
# (By.CLASS_NAME, 'suggest2-form__button')
# ))
# box.send_keys(query)
# button.click()
# except TimeoutException:
# print('Box or Button didnt find')
sent = '"' + query[0] + '"'
box = driver.find_element_by_name('text')
# button = driver.find_element_by_class_name('suggest2-form__button')
# button = driver.find_element_by_class_name('button mini-suggest')
button = driver.find_element_by_class_name('search2__button')
box.send_keys(sent)
button.click()
yandex_dict[sent] = driver.page_source
for sent in query[1:]:
close = driver.find_element_by_class_name('input__clear')
close.click()
box = driver.find_element_by_name('text')
button = driver.find_element_by_class_name('websearch-button')
sent = '"' + sent + '"'
box.send_keys(sent)
button.click()
yandex_dict[sent] = driver.page_source
html = driver.page_source
driver.close()
if w:
print(1)
with open('index.html', 'w') as f:
f.write(html)
return yandex_dict
def getFloData(outFileDir):
profile=FirefoxProfile()
profile.set_preference('browser.download.folderList', 2)
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv')
profile.set_preference('browser.download.dir', outFileDir)
opts = Options()
opts.set_headless()
assert opts.headless
browser = Firefox(options=opts, firefox_profile=profile)
browser.get('https://user.meetflo.com/login')
assert(browser.title == 'Flo - My Account')
FloUser = os.environ['FLO_USER']
FloPW = os.environ['FLO_PW']
login_form=browser.find_elements_by_class_name('form-control')
login_form[0].send_keys(FloUser)
login_form[1].send_keys(FloPW)
login_button=browser.find_elements_by_class_name('btn')
login_button[0].click()
try:
element = WebDriverWait(browser, 10).until(
EC.title_is('Flo - Home'))
finally:
pass
browser.get('https://user.meetflo.com/usage')
try:
element = WebDriverWait(browser, 10).until(
EC.title_is('Flo - Usage'))
finally:
pass
assert(browser.title == 'Flo - Usage')
dataDownload = browser.find_element_by_class_name('data-download')
# pdb.set_trace()
# why this sleep should be necessary is a mystery
time.sleep(2)
dataDownloadButton = dataDownload.find_element_by_class_name('dropdown-toggle')
dataDownloadButton.click()
csv_button = None
dropdownItemList=browser.find_elements_by_class_name('dropdown-item')
for (i,item) in enumerate(dropdownItemList):
if DEBUG:
print(i, item.get_attribute('outerHTML'))
print(item.text)
if item.text=='as CSV':
csv_button = item
assert(csv_button.text=='as CSV')
csv_button.click()
browser.close()
def __init__(self):
options = Options()
options.set_headless(headless=True)
self.driver = webdriver.Firefox(firefox_options=options)
self.driver.set_page_load_timeout(60)
def retrieve_squad_national_team(confederation):
if confederation == 'UEFA':
opts = Options()
opts.set_headless = True
driver = webdriver.Firefox(options=opts)
driver.get(urls['url_1'])
# vereinprofil_tooltip tooltipstered
POSITION_TEAM = 0
NAME_TEAM = 1
TOTAL_VALUE_TEAM = 4
CONFEDERATION_TEAM = 5
#country_buttons = driver.find_elements_by_xpath("//a[@class='vereinprofil_tooltip tooltipstered']")
#country_buttons = driver.find_elements_by_xpath("//td[@class='hauptlink']")
#26-50
pager = driver.find_elements_by_xpath(
"//div[@class='pager']/ul[@id='yw2']/li/a")
time.sleep(3)
#pager[PAGE_2].click()
for page_number in range(0, 9):
#if page_number == 0 or page_number == 1:
driver.execute_script("arguments[0].click();", pager[page_number])
#else:
# driver.execute_script("arguments[0].click();", pager[page_number + 2])
time.sleep(3)
pager = driver.find_elements_by_xpath(
"//div[@class='pager']/ul[@id='yw2']/li/a")
#print(pager[page_number].get_attribute('outerHTML'))
table_teams = driver.find_elements_by_xpath("//tbody/tr")
# PAGE 1: 1-25
#print('len:' + str(len(table_teams)))
number_of_teams_on_the_page = len(table_teams)
for k in range(number_of_teams_on_the_page - 1,
number_of_teams_on_the_page):
#print('table_teams = ' + str(table_teams[k].get_attribute('innerHTML')))
team = table_teams[k].find_elements_by_tag_name('td')
#print(team[NAME].get_attribute('innerHTML'))
#print('position = ' + str(rows[POSITION].get_attribute('innerHTML')))
print('position = ' + str(team[POSITION_TEAM].text))
print('-')
#print('******' + str(team[POSITION].get_attribute('outerHTML')))
name = team[NAME_TEAM].find_elements_by_tag_name('a')
#print('name = ' + str(name))
#print('************** ' + str(name[0].get_attribute('innerHTML')))
#print('name = ' + str(rows[NAME].get_attribute('innerHTML')))
print('name = ' + str(name[1].get_attribute('innerHTML')))
print('-')
print('total_value = ' +
str(team[TOTAL_VALUE_TEAM].get_attribute('innerHTML')))
print('-')
print('confederation = ' +
str(team[CONFEDERATION_TEAM].get_attribute('innerHTML')))
#click on team's name
driver.execute_script("arguments[0].click();", name[1])
time.sleep(3)
#######################################################################
# squad data
#######################################################################
#market_value = driver.find_elements_by_xpath("//div[@class='dataMarktwert']/a/span[@class='waehrung']")
# market_value = driver.find_elements_by_xpath("//div[@class='dataMarktwert']/a")
# print('market_value = ' + str(market_value[0].text))
# formation
formation_row = driver.find_elements_by_xpath(
"//div[@class='large-7 aufstellung-vereinsseite columns small-12 unterueberschrift aufstellung-unterueberschrift']"
)
#print(formation_row[0])
formation_row = formation_row[0].get_attribute('innerHTML')
formation_row = formation_row.rstrip().lstrip()
#print('formation row: ' + str(formation_row))
formation = formation_row.split(' ') #
# formation = formation[2]
print('formation: ' + formation[2])
# label substitutes and players
squad = driver.find_elements_by_xpath(
"//div[@class='large-7 columns small-12 aufstellung-vereinsseite']/div[@style='position: absolute; top: 0; bottom: 0; left: 0; right: 0;']/div[@class='aufstellung-spieler-container']"
)
#squad = driver.find_elements_by_xpath("//div[@class='large-7 columns small-12 aufstellung-vereinsseite']")
#print("squad: " + squad[0].get_attribute('innerHTML'))
#squad =
counter = 0
name_and_numbers = []
dictionary = {}
for s in squad:
# print(str(counter) + ") " + str(s.text))
st = s.text
name_number = st.split('\n')
number = name_number[0]
name = name_number[1]
dictionary = {}
dictionary['number'] = number
dictionary['name'] = name
name_and_numbers.append(dictionary) #
counter += 1
#print(name_and_numbers)
#print(name_and_numbers)
print("************************Squad*******************")
for name_and_number in name_and_numbers:
print(
str(name_and_number['number']) + " " +
str(name_and_number['name']))
#######################################################################
#######################################################################
# substitutes data
#######################################################################
substitutes = driver.find_elements_by_xpath(
"//div[@class='large-5 columns small-12 aufstellung-ersatzbank-box aufstellung-vereinsseite']/table[@class='ersatzbank']/tbody/tr"
)
name_and_numbers_sub = []
dictionary_sub = {}
print(
"***************************Substitutes***********************"
)
for subs in substitutes:
sub = subs.find_elements_by_tag_name('td')
#print(str(sub[0].text) + " " + str(sub[1].text))
# st = sub.text
# name_number = st.split('\n')
sub_number = sub[0].text
sub_name = sub[1].text
# print(sub_number)
# print(sub_name)
dictionary_sub = {}
dictionary_sub['number'] = sub_number
dictionary_sub['name'] = sub_name
name_and_numbers_sub.append(dictionary_sub) #
# print(sub[0].text)
# print(sub[1].text)
# print('-------------------------------')
#!!! last one is the manager, so don't append it to the dictionary_sub
#print(name_and_numbers_sub)
for name_and_number_sub in name_and_numbers_sub:
number = name_and_number_sub['number']
name = name_and_number_sub['name']
print(str(number) + " " + str(name))
#######################################################################
# manager data
#######################################################################
#slider-list
table_managers = driver.find_elements_by_xpath(
"//div[@class='large-4 columns']/div[@class='box box-slider']/div[@class='clearer']"
)
table_managers = driver.find_elements_by_xpath(
"//div[@class='container-inhalt']/div[@class='container-hauptinfo']/a"
)
manager_name = table_managers[0].get_attribute('innerHTML') #
print("manager_name: " + str(manager_name))
driver.execute_script("arguments[0].click();",
table_managers[0])
time.sleep(4)
manager_info = driver.find_elements_by_xpath(
"//div[@id='trainer_head']/div[@class='large-12 columns']/div[@class='dataHeader']/div[@class='dataContent']"
)
manager_info = manager_info[0].find_elements_by_tag_name(
'span')
# for info in manager_info:
# print(str(counter) + ") " + str(info.get_attribute('innerHTML')))
# counter += 1
manager_date_of_birth = manager_info[1].get_attribute(
'innerHTML')
manager_place_of_birth = manager_info[4].get_attribute(
'innerHTML')
manager_citizenship = manager_info[7].get_attribute(
'innerHTML')
manager_preferred_formation = manager_info[17].get_attribute(
'innerHTML')
manager_date_of_birth = re.sub(' +', ' ',
manager_date_of_birth)
m = manager_date_of_birth.split()
manager_date_of_birth_month = m[0] #
manager_date_of_birth_day = m[1].replace(',', '') #
manager_date_of_birth_year = m[2] #
manager_date_of_birth_age = m[3].replace('(',
'').replace(')',
'') #
print("manager_date_of_birth: ")
print(" month: " + str(manager_date_of_birth_month))
print(" day: " + str(manager_date_of_birth_day))
print(" year: " + str(manager_date_of_birth_year))
print(" age: " + str(manager_date_of_birth_age))
print("manager_place_of_birth: " + str(manager_place_of_birth))
print("manager_citizenship: " + str(manager_citizenship))
print("manager_preferred_formation: " +
str(manager_preferred_formation))
#print('manager_info: ' + manager_info[0].get_attribute('innerHTML'))
# get manager's photo
manager_info = driver.find_elements_by_xpath(
"//div[@id='trainer_head']/div[@class='large-12 columns']/div[@class='dataHeader']/div[@class='dataBild']"
)
print('************************Photo')
manager_photo = manager_info[0].find_elements_by_tag_name(
'img')[0].get_attribute('src')
print("manager_photo: " + str(manager_photo))
#print(manager_info[0].get_attribute('innerHTML'))
driver.back()
time.sleep(4)
######################################################################
#table_players = driver.find_elements_by_xpath("//div[class='responsive-table']/div[@class='grid-view']/table[@class='items']/tbody/tr")
table_players = driver.find_elements_by_xpath(
"//table[@class='items']/tbody/tr")
squad_size = len(table_players)
#print('***** ' + str(len(table_players)))
######################################################################
######################################################################
for i in range(squad_size - 1, squad_size):
#print(i)
print('------------------------------------------')
print('------------------------------------------')
print('------------------------------------------')
player = table_players[i].find_elements_by_tag_name('td')
NATIONAL_TEAM_NUMBER = 0
POSITION = 4
NAME = 5
MARKET_VALUE = 8
national_team_number = player[NATIONAL_TEAM_NUMBER].text
position = player[POSITION].get_attribute('innerHTML')
name = player[NAME].get_attribute('innerHTML')
market_value = player[MARKET_VALUE].text
print('number: ' + str(national_team_number))
print('position: ' + str(position))
print('name: ' + str(name))
print('market value: ' + str(market_value))
player_link = player[1].find_elements_by_xpath(
"//table[@class='inline-table']/tbody/tr/td[@class='hauptlink']/div/span/a"
)
#print('link: ' + str(player_link[0].get_attribute('href')))
#print(player[1].get_attribute('outerHTML'))
driver.execute_script("arguments[0].click();",
player_link[2 * i])
time.sleep(4)
#######################################################################
# player data
#######################################################################
player_data = driver.find_elements_by_xpath(
"//div[@class='row']/div[@class='large-12 columns']/div[@class='dataHeader dataExtended']/div"
)
#print(player_data[2].get_attribute('outerHTML'))
#player_data[2].find_elements_by_tag_name('img')
# player image
player_image = player_data[2].find_elements_by_tag_name(
'img')[0].get_attribute('src') #
#print(player_data[2].find_elements_by_tag_name('img')[0].get_attribute('src'))
print('player_image: ' + str(player_image))
#####################################################################
# player date_of_birth
player_info = player_data[1].find_elements_by_xpath(
"//div[@class='dataBottom']")
player_dob = player_info[0].find_elements_by_tag_name(
'span')
player_date_of_birth = player_dob[1].get_attribute(
'innerHTML').rstrip().lstrip()
player_date_of_birth = re.sub(' +', ' ',
player_date_of_birth)
p = player_date_of_birth.split()
player_date_of_birth_month = p[0] #
player_date_of_birth_day = p[1].replace(',', '') #
player_date_of_birth_year = p[2] #
player_date_of_birth_age = p[3].replace('(', '').replace(
')', '') #
print('month: ' + player_date_of_birth_month)
print('day: ' + player_date_of_birth_day)
print('year: ' + player_date_of_birth_year)
print('age: ' + player_date_of_birth_age)
#print(player_date_of_birth)
#print(player_dob[1].get_attribute('innerHTML').rstrip().lstrip())
#####################################################################
# player place_of_height
#print(player_info[0].get_attribute('innerHTML'))
player_height = player_info[0].find_elements_by_tag_name(
'span')
if player_height[8].get_attribute(
'innerHTML') == 'Height:':
height = player_height[9].get_attribute('innerHTML') #
print('height: ' + str(height))
#print(player_info[0])
#player_height = player_info[0]
#####################################################################
# player total value
# player_info = player_data[4].find_elements_by_tag_name('a')
# #player_total_value = player_info[0].find_elements_by_tag_name('span')
# #player_total_value = player_info[0].get_attribute('innerHTML')
# player_total_value = player_info[0].text
# player_tot_val_list = player_total_value.split(' ')
# player_total_value = player_tot_val_list[0].split('\n')
# player_total_value = player_total_value[0] #
# print('total_value: ' + str(player_total_value))
#print('*** ' + str(player_total_value[1].get_attribute('innerHTML')))
#####################################################################
# place of birth
player_info = player_data[1].find_elements_by_xpath(
"//div[@class='dataBottom']/div")
player_place_of_birth = player_info[
0].find_elements_by_tag_name('span')
#if player_place_of_birth[3]
if player_place_of_birth[2].get_attribute(
'innerHTML') == 'Place of birth:':
player_place_of_birth_city = player_place_of_birth[
4].get_attribute('innerHTML') #
print('place_of_birth: ' +
str(player_place_of_birth_city))
driver.back()
time.sleep(4)
table_players = driver.find_elements_by_xpath(
"//table[@class='items']/tbody/tr")
print('********************************')
driver.back()
time.sleep(4)
#pager = driver.find_elements_by_xpath("//div[@class='pager']/ul[@id='yw2']/li/a")
#print(pager[page_number].get_attribute('outerHTML'))
table_teams = driver.find_elements_by_xpath("//tbody/tr")
#team = table_teams[k + 1].find_elements_by_tag_name('td')
######################################################################
######################################################################
driver.back()
time.sleep(4)
pager = driver.find_elements_by_xpath(
"//div[@class='pager']/ul[@id='yw2']/li/a")
time.sleep(3)
def main():
"""Entrypoint for blobtools view."""
args = docopt(__doc__)
loc = "%s/%s/dataset/%s" % (args['--host'], args['--prefix'],
args['DATASET'])
qstr = "staticThreshold=Infinity"
qstr += "&nohitThreshold=Infinity"
qstr += "&plotGraphics=svg"
if args['--format'] == 'svg':
qstr += "&svgThreshold=Infinity"
shape = 'square'
for param in args['--param']:
qstr += "&%s" % str(param)
key, value = param.split('=')
if key == 'plotShape':
shape = value
timeout = int(args['--timeout'])
outdir = os.path.abspath(args['--out'])
os.makedirs(Path(outdir), exist_ok=True)
profile = webdriver.FirefoxProfile()
profile.set_preference('browser.download.folderList', 2)
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', outdir)
profile.set_preference('browser.download.lastDir', args['--out'])
profile.set_preference(
'browser.helperApps.neverAsk.saveToDisk',
'image/png, image/svg+xml, text/csv, text/plain, application/json')
options = Options()
options.set_headless(headless=False)
display = Display(visible=0, size=(800, 600))
display.start()
driver = webdriver.Firefox(options=options, firefox_profile=profile)
try:
view = args['--view'][0]
if args['--preview']:
qstr += '#Filters'
url = "%s/%s?%s" % (loc, view, qstr)
print("Loading %s" % url)
try:
driver.get(url)
except Exception as err:
print(err)
for next_view in args['--view']:
if next_view != view:
view = next_view
url = "%s/%s?%s" % (loc, view, qstr)
print("Navigating to %s" % url)
try:
driver.get(url)
except Exception as err:
print(err)
for fmt in args['--format']:
file = "%s.%s" % (args['DATASET'], view)
if view == 'blob':
file += ".%s" % shape
elif view == 'busco':
view = "all_%s" % view
if fmt not in ('csv', 'json'):
fmt = 'json'
file += ".%s" % fmt
print("Fetching %s" % file)
el_id = "%s_save_%s" % (view, fmt)
print("waiting for element %s" % el_id)
unstable = True
while unstable:
try:
element = WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.ID, el_id)))
element.click()
unstable = False
file_name = "%s/%s" % (outdir, file)
print("waiting for file '%s'" % file_name)
file_ready(file_name)
except Exception as err:
time.sleep(1)
for preview in args['--preview']:
print("Creating %s preview" % preview)
for fmt in args['--format']:
el_id = "%s_preview_save_%s" % (preview, fmt)
file = "%s.%s.preview.%s" % (args['DATASET'], preview, fmt)
try:
element = WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.ID, el_id)))
element.click()
file_name = "%s/%s" % (outdir, file)
print("waiting for file '%s'" % file_name)
file_ready(file_name)
except Exception as err:
print(err)
driver.close()
except Exception as err:
print(err)
driver.close()
def driver(self):
options = Options()
options.set_headless(headless=True)
return webdriver.Firefox(firefox_options=options)
def __init__(self):
firefox_options = Options()
firefox_options.set_headless()
self.browser = Firefox(options=firefox_options)
self.browser = webdriver.Firefox()
self.browser.get("https://mbasic.facebook.com/login")
print("Initializing...")
import os
import wikipedia
import datetime
from selenium import webdriver as wd
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.common.action_chains import ActionChains
import enchant
print("Opening Browser For Testing...")
driver = wd.Firefox(executable_path=GeckoDriverManager().install())
os.system("clear")
# driver.close()
opt = Options()
opt.set_headless()
assert opt
headless_driver = wd.Firefox(executable_path=GeckoDriverManager().install(),options=opt)
# headless_driver.close()
print("Looking Good. Clearing Terminal.")
os.system("clear")
dictionary = enchant.Dict("en_UK")
def search_app(channel=""):
channel=input("Which Video do you want to view :")
driver.get("https://www.youtube.com/")
search_ele = driver.find_element_by_name("search_query")
search_ele.send_keys(channel,Keys.ENTER)
def searchQuestion(query):
driver.get("https://duckduckgo.com/")
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = Options()
options.set_headless(headless=False)
driver = webdriver.Firefox(firefox_options=options)
is_last = False
nr_art = 0
nr_rev = 0
pagina = 'https://www.ncbi.nlm.nih.gov/pubmed/?term=Br+J+Anaesth.+2018+Jul'
#Para max_resultados()
id_maxres = "Display"
id_index = 2
def obter_pagina(pagina):
driver.get(pagina)
print("Headless Firefox Initialized")
def max_resultados(id_maxres, id_index):
lista = driver.find_elements_by_id(id_maxres)
lista[id_index].click()
driver.find_element_by_id("ps200").click()
def sacar():
def setUp(self):
options = Options()
options.set_headless(headless=True)
self.browser = webdriver.Firefox(firefox_options=options)
self.browser.implicitly_wait(1)
def load_page(url):
# Create a filename for the output files
file_name = url.lower().replace("http://", "").replace(
"https://",
"").replace("www.", "").split("/")[0].replace("-",
"_").replace(".", "_")
#Initialize a server/proxy for loading the page
server = Server(browsermob_location)
server.start()
profile = webdriver.FirefoxProfile()
#profile.accept_untrusted_certs = True
#profile.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0")
opts = Options()
opts.set_headless()
assert opts.headless
driver = webdriver.Firefox(options=opts,
firefox_profile=profile,
executable_path=geckodriver_location)
# Try to load the page
try:
driver.get(url)
except Exception as e:
print("Error during load of {}: {}".format(url, str(e).strip()))
# Be sure you try HTTP if HTTPS fails
if "https://" not in url:
url = url.replace("http://", "https://")
driver.get(url)
# Parse the HTML
try:
print("Getting HTML")
with open('{}/{}.html'.format(output_dir, file_name),
'w+',
encoding='utf-8') as save_html:
save_html.write(str(driver.page_source))
except Exception as e:
print("Error getting page source: {}".format(str(e)))
# Get a screenshot with static size as pages will have different formats.
try:
print("Getting Screenshot")
driver.set_window_size(1920, 2000)
driver.save_screenshot('{}/{}.png'.format(output_dir, file_name))
except Exception as e:
print("Error getting screenshot: {}".format(str(e)))
# Get the har data (e.g. resources it loads)
try:
print("Getting HAR data")
proxy = server.create_proxy()
profile.set_proxy(proxy.selenium_proxy())
driver = webdriver.Firefox(options=opts,
firefox_profile=profile,
executable_path=geckodriver_location)
driver.get(url)
proxy.new_har(url, options={'captureHeaders': True})
har_data = json.dumps(proxy.har, indent=4)
with open('{}/{}.har'.format(output_dir, file_name), 'w+') as save_har:
save_har.write(str(har_data))
except Exception as e:
print("Error getting har: {}".format(str(e).strip()))
# Kill everything...
server.stop()
driver.quit()
print("Done!")
from selenium import webdriver from selenium.webdriver.firefox.options import Options import os import time #open Firefox browser headless opts = Options() opts.set_headless(headless=True) assert opts.headless # no log file setting browser = webdriver.Firefox(options=opts, service_log_path=os.devnull) # service_log_path = os.devnull makes no geckodriver.log file url = 'https://www.google.com.tw/' browser.get(url) time.sleep(3) browser.quit()
def main():
p = subprocess.Popen(["darkreader/host_with_flask.py"],
stdout=sys.stdout,
stderr=sys.stderr)
# while True:
# stdout = str(p.stdout.readline(), encoding="UTF-8")
# print("stdout:", stdout)
# if "* Running on http://" in stdout:
# break
# url = "http://" + stdout.split("http://")[1].split("/")[0]
url = "http://127.0.0.1:5687/"
# print(stdout, url)
time.sleep(4)
# print("received:", requests.get(url).text)
options = Options()
options.set_headless(headless=True)
driver = webdriver.Firefox(firefox_options=options)
js = """
var callback = arguments[0];
DarkReader.exportGeneratedCSS().then(function(result){callback(result);});
"""
# Generate darkreader css for index.html:
driver.get(url)
time.sleep(3)
darkreader_generated_css = driver.execute_async_script(js).replace(
url, "https://phseiff.com/")
with open("index.html.darkreader.css", "w") as f:
f.write(darkreader_generated_css)
with open("index.html", "r") as f:
content = f.read()
content = content.replace(
"""
<script type="text/javascript" src="darkreader/darkreader.min.js"></script>
<script>
DarkReader.setFetchMethod(window.fetch);
DarkReader.enable({ // <-- always use darkmode instead of, like previously, adapting to the system.
brightness: 100,
contrast: 90,
sepia: 10
});
</script>""", "<style>" + css_html_js_minify.css_minify(
darkreader_generated_css.split("/* Modified CSS */")[0]) +
"</style>")
content = content.replace(
"""
<!-- Ensure dark reader is executed again after we updated the essays using javascript. -->
<script>
DarkReader.disable();
DarkReader.enable({ // <-- always use darkmode instead of, like previously, adapting to the system.
brightness: 100,
contrast: 90,
sepia: 10
});
</script>""",
"""<link type="text/css" rel="stylesheet" href="/index.html.darkreader.min.css">"""
)
with open("index.html", "w") as f:
f.write(content)
# Generate darkreader css for github card:
driver.get(url + "//github-card/response.html")
darkreader_generated_css = driver.execute_async_script(js).replace(
url, "https://phseiff.com")
with open("github-card/response.html.darkreader.css", "w") as f:
f.write(darkreader_generated_css.split("/* Modified CSS */")[1])
with open("github-card/response.html", "r+") as f:
content = f.read()
content = content.replace(
"""
<script type="text/javascript" src="../darkreader/darkreader.min.js"></script>
<script>
DarkReader.setFetchMethod(window.fetch)
DarkReader.enable({
brightness: 100,
contrast: 90,
sepia: 10
});
</script>""",
"""<link type="text/css" rel="stylesheet" href="/github-card/response.html.darkreader.min.css">"""
)
with open("github-card/response.modified.html", "w") as f:
f.write(content)
print("got all but error page!")
# Generate css for 404 error page:
driver.get(url + "//404-raw.html")
darkreader_generated_css = driver.execute_async_script(js).replace(
url, "https://phseiff.com")
with open("404.html.darkreader.css", "w") as f:
f.write(darkreader_generated_css)
with open("404-raw.html", "r") as f:
with open("404.html", "w") as f2:
f2.write(f.read().replace(
"""
<script type="text/javascript" src="/darkreader/darkreader.min.js"></script>
<script>
DarkReader.setFetchMethod(window.fetch);
DarkReader.enable({
brightness: 100,
contrast: 90,
sepia: 10
});
</script>""",
("""<link type="text/css" rel="stylesheet" href="/404.html.darkreader.min.css">"""
if False else
"""<link type="text/css" rel="stylesheet" href="/index.html.darkreader.min.css">"""
)))
p.kill()
driver.close()
def scrape_dea(zips):
'''
Purpose: Function that will scrape dropbox locations for all zip
codes contained in the list `zips`.
Input: zips (list): list of zip codes
Output: CSV file containing all unique dropbox locations for the
zip codes specified in `zips`
'''
# Open a Headless Firefox web browser and direct it to the DEA's dropbox
# search page
options = Options()
options.set_headless(headless=True)
browser = webdriver.Firefox(firefox_options=options)
# browser = webdriver.Firefox()
browser.get('https://apps.deadiversion.usdoj.gov/SEARCH-NTBI')
browser.implicitly_wait(10)
# final storage container for dropbox locations
dropboxList = []
count = 0
# For every zip code in the US, run the dropbox location search on the site
for code in zips:
status = ''
status = scrape_page(browser, code)
# If we find a bad zip code or no dropbox locations are found in a
# zip, skip to the next zip code
if status == 'NONE_FOUND' or status == 'BAD_ZIP':
browser.get('https://apps.deadiversion.usdoj.gov/SEARCH-NTBI')
continue
else:
# For every column in a row in the dropbox location table, grab
# the data and place it in the list `rowList`. After each
# row is read, add that data to the master list `dropboxList`.
for tr in status.findAll('tr')[2:-1]:
rowList = []
for td in tr.findAll('td')[:4]:
# Removes special characters from any entries
# text = td.text.encode('utf-8').replace('\xd1',"'").replace(
# '\xbf',"'").decode('ascii','ignore').strip()
text = td.text.strip()
rowList.append(' '.join(text.split()))
# Convert state, zip to separate columns
stateAndZip = tr.findAll('td')[4]
rowList.append(stateAndZip.text.split(', ')[0])
rowList.append(stateAndZip.text.split(', ')[1])
# Convert map data to a google link
mapURL = 'http://maps.google.com/maps?q=' + tr.find(
'a', href=True)['href'].split('daddr=')[1]
rowList.append(mapURL)
dropboxList.append(rowList)
# Deduplicate the list whenever it exceeds a size greater than 500
if len(dropboxList) > 500:
dropboxList = dedup_list(dropboxList)
# Print status update after every 25 zip codes have been scraped
count += 1
if count % 25 == 0:
print('Completed ' +
str(round(((zips.index(code) / len(zips)) * 100), 2)) +
'% of list')
# After scraping every 100 zip codes, export the list to a csv file
if count % 100 == 0:
print('Printing file: ' + code)
# Convert storage container into pandas dataframe
dropboxDF = pd.DataFrame(dropboxList,
columns=[
'Participant', 'CollectionSite',
'Address', 'City', 'State', 'Zip',
'MapURL'
])
# Remove duplicate entries caused by querying nearby zip codes
dropboxDF.drop_duplicates(inplace=True)
# Export dataframe to CSV file in the working directory
filename = (
'Data/October2018_NTBI/dropbox_addresses_October2018_' +
str(code) + '.csv')
dropboxDF.to_csv(filename, index=False)
dropboxList = []
dropboxDF = pd.DataFrame()
browser.back()
# Output the final batch of locations
dropboxDF = pd.DataFrame(dropboxList,
columns=[
'Participant', 'CollectionSite', 'Address',
'City', 'State', 'Zip', 'MapURL'
])
# Remove any duplicate entries caused by querying nearby zip codes
dropboxDF.drop_duplicates(inplace=True)
# Export dataframe to CSV file in the working directory
filename = ('Data/October2018_NTBI/dropbox_addresses_October2018_' +
str(code) + '.csv')
dropboxDF.to_csv(filename, index=False)
# Close the browser
browser.close()
return
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options as FFOptions
from selenium.webdriver.chrome.options import Options as CHOptions
#assert "Python" in driver.title
#elem = driver.find_element_by_name("q")
#elem.clear()
#elem.send_keys("pycon")
#elem.send_keys(Keys.RETURN)
#assert "No results found." not in driver.page_source
ff_options = FFOptions()
ff_options.set_headless(True)
ch_options = CHOptions()
ch_options.set_headless(True)
#chrome = webdriver.Chrome(chrome_options = ch_options)
firefox = webdriver.Firefox(firefox_options=ff_options)
#phantom = webdriver.PhantomJS()
for i in range(100):
#firefox.get("http://www.acfun.cn/")
#print(firefox.title)
#chrome.get("https://cn.bing.com/")
firefox.get("https://www.baidu.com/")
print(firefox.title, i)
#chrome.close()
firefox.close()
#phantom.close()
def EventData(event_id):
url = 'https://www.facebook.com/events/'+event_id
try:
fp = urllib.request.urlopen(url)
mybytes = fp.read()
mystr = mybytes.decode("utf8")
fp.close()
except:
logs.print2("Couldn't load page")
mystr = "null"
#########gets event name
try:
text = mystr.split('data-testid="event-permalink-event-name">')[1]
event_name = text.split('</h1')[0]
if """ in event_name:
event_name = event_name.replace(""", '"')
if "'" in event_name:
event_name = event_name.replace("'", "'")
except:
logs.print2("Couldn't extract event name,god damn it facebook")
event_name = "Couldn't extract event name"
#########gets event date
try:
text = mystr.split('<span itemprop="startDate">')[1]
text_left = text.split('</span>')[0]
text_left = text_left+" "
number_length = 4
pattern= r"\D(\d{%d})\D" % number_length
test = re.findall(pattern, text_left)
if not test:
now = datetime.datetime.now()
event_date = text_left+" " + str(now.year)
else:
event_date = text_left
except:
logs.print2("Couldn't extract event date,god damn it facebook")
event_date = "Couldn't extract event date"
#########gets event description
options = Options()
options.set_headless(headless=True)
driver = webdriver.Firefox(firefox_options=options)
try:
driver.get(url)
except:
logs.print2("Couldn't load page for event description")
try:
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'uiContextualLayerParent')))
time.sleep(20)
except TimeoutException:
print('Page timed out after 10 secs.')
try:
soup = BeautifulSoup(driver.page_source, 'html.parser')
driver.quit()
text = soup.text.split('φίλους σας')[1]
event_description = text.split('Δείτε τη συζήτηση')[0]
event_description = event_description.split('Λεπτομέρειες')[1]
event_description = event_description.split('Δείτε περισσότεραΠληροφορίες')[0]
except:
logs.print2("Couldn't extract event description,god damn it facebook")
event_description = "Couldn't extract event description"
return (event_description,event_date,event_name)
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary('/home/nikolai/ff/firefox/firefox')
options = Options()
options.set_headless(headless=True)
driver = webdriver.Firefox(firefox_binary=binary,
firefox_options=options, executable_path='../Drivers/geckodriver')
driver.get("http://google.com/")
print ("Headless Firefox Initialized")
driver.quit()
def _setup_firefox():
opts = Options()
opts.set_headless()
assert opts.headless
browser = Firefox(executable_path=GECKODRIVER_PATH, options=opts)
return browser
def __init__(self, url):
try:
firefox_options = Options()
firefox_options.set_headless()
user_agent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"
profile = webdriver.FirefoxProfile()
profile.set_preference("general.useragent.override", user_agent)
executable_path = './drivers/geckodriver.exe'
if sys.platform.startswith('linux'):
executable_path = './drivers/geckodriver'
self.browser = webdriver.Firefox(profile,
options=firefox_options,
executable_path=executable_path)
self.browser.set_window_size(360, 640)
self.browser.get(url)
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"//span[contains(text(), 'Help your friend slash the price！')]"
))).click()
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located(
(By.XPATH,
r"/html/body/div[1]/div/div[2]/div[1]/div/ul/li[1]/div"
))).click()
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"/html/body/div[1]/div/div[2]/div[2]/div[1]/div/div/div/div[4]/div[1]/input"
))).send_keys(
str(random.randint(111111111111, 999999999999)) +
"@gmail.com")
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"/html/body/div[1]/div/div[2]/div[2]/div[1]/div/div/div/div[4]/div[2]/input"
))).send_keys("Ali" + str(random.randint(11111, 99999)) +
"Conta" + str(random.randint(11111, 99999)))
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"/html/body/div[1]/div/div[2]/div[2]/div[1]/div/div/div/div[4]/div[5]/a"
))).click()
try:
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located(
(By.XPATH, r"//a[contains(text(), 'CONFIRA!')]")))
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located(
(By.XPATH, r"//span[@id='ms-back']"))).click()
try:
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"//span[contains(text(), 'Ajude seu amigo a pechinchar')]"
))).click()
try:
WebDriverWait(self.browser, 30).until(
EC.presence_of_element_located((
By.XPATH,
r"//div[contains(text(), 'Você acabou de cortar')]"
)))
print(
self.tipo_mensagem("sucesso",
u"Ajudado com sucesso."))
time.sleep(2)
self.browser.close()
self.finalizar()
except NoSuchElementException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except ElementClickInterceptedException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except ElementNotInteractableException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except NoSuchElementException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except ElementClickInterceptedException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except ElementNotInteractableException:
print(
self.tipo_mensagem("erro",
u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except NoSuchElementException:
print(
self.tipo_mensagem(
"erro",
u"Não foi possivel encontrar os elementos na página."))
self.browser.close()
self.finalizar()
except ElementClickInterceptedException:
print(
self.tipo_mensagem(
"erro",
u"Não foi possivel encontrar os elementos na página."))
self.browser.close()
self.finalizar()
except ElementNotInteractableException:
print(
self.tipo_mensagem(
"erro",
u"Não foi possivel encontrar os elementos na página."))
self.browser.close()
self.finalizar()
except NoSuchElementException:
print(
self.tipo_mensagem(
"erro",
u"Não foi possivel encontrar os elementos na página."))
self.browser.close()
self.finalizar()
except ElementClickInterceptedException:
print(self.tipo_mensagem("erro", u"Não foi possivel Ajudar."))
self.browser.close()
self.finalizar()
except ElementNotInteractableException:
print(
self.tipo_mensagem(
"erro",
u"Não foi possivel encontrar os elementos na página."))
self.browser.close()
self.finalizar()
except TimeoutException:
print(
self.tipo_mensagem(
"erro", u"Não foi possivel carregar a página de login."))
self.browser.close()
self.finalizar()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('yaml_settings')
args = parser.parse_args()
time = datetime.utcnow()
with open(args.yaml_settings, 'r') as fh:
yaml_settings = yaml.safe_load(fh)
options = Options()
options.set_headless(headless=True)
driver = webdriver.Firefox(firefox_options=options)
driver.implicitly_wait(150)
try:
driver.get("https://fugu.anon.edu/grafana/login/")
driver.find_element_by_name("username").click()
driver.find_element_by_name("username").clear()
driver.find_element_by_name("username").send_keys("fugu")
driver.find_element_by_id("inputPassword").click()
driver.find_element_by_id("inputPassword").clear()
driver.find_element_by_id("inputPassword").send_keys(GRAFANA_PWD)
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Help'])[1]/following::button[1]")
driver.find_element_by_xpath(xpath).click()
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Help'])[1]/following::i[5]")
driver.find_element_by_xpath(xpath).click()
driver.find_element_by_link_text("Snapshot").click()
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Expire'])[1]/following::select[1]")
driver.find_element_by_xpath(xpath).click()
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Expire'])[1]/following::select[1]")
Select(driver.find_element_by_xpath(xpath)).select_by_visible_text(
"1 Hour")
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Expire'])[1]/following::select[1]")
driver.find_element_by_xpath(xpath).click()
# begin code to set timeout
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Timeout (seconds)'])[1]/following::input[1]")
driver.find_element_by_xpath(xpath).click()
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Timeout (seconds)'])[1]/following::input[1]")
driver.find_element_by_xpath(xpath).clear()
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Timeout (seconds)'])[1]/following::input[1]")
driver.find_element_by_xpath(xpath).send_keys("60")
# end code to set timeout
xpath = ("(.//*[normalize-space(text()) and normalize-space(.)"
"='Timeout (seconds)'])[1]/following::button[1]")
driver.find_element_by_xpath(xpath).click()
prefix = "https://fugu.anon.edu/grafana/dashboard/snapshot/"
snapshot_url = driver.find_element_by_partial_link_text(prefix).text
sys.stderr.write("Generated snapshot: {}\n".format(snapshot_url))
driver.quit()
except NoSuchElementException:
driver.quit()
sys.exit("Error generating snapshot")
# Now, add this link to postgres, and delete old links from the table
conn = None
try:
conn = connect_to_postgres(yaml_settings)
cur = conn.cursor()
add_snap_cmd = ("INSERT INTO fugu_grafanasnapshot "
"(url, created_on) VALUES (%s, %s)")
cur.execute(add_snap_cmd, (snapshot_url, time))
del_old_snap_cmd = "DELETE FROM fugu_grafanasnapshot WHERE (url) != (%s)"
cur.execute(del_old_snap_cmd, (snapshot_url, ))
conn.commit()
cur.close()
except:
sys.stderr.write("Failed to post data to PostgreSQL\n")
finally:
if conn is not None:
conn.close()
#asset "arizona" in driver.title
#==========================================
#=========================================================#
if __name__ == "__main__":
#========== Subprocess pckg (call) to get tor to work =========#
#=========== Adding ad nauseam extentesion =============#
options = Options()
options.set_headless(True)
geckodriver = '/Users/MyMac/Desktop/finalprojectesoc/geckodriver'
driver = webdriver.Firefox(executable_path=geckodriver, firefox_options=options)
wait = WebDriverWait(driver, timeout=10)
addon1 = '/Users/MyMac/Library/Application Support/Firefox/Profiles/i0nhni3z.default/extensions/[email protected]'
driver.install_addon(addon1, temporary=True)
import random
import time
import asyncio
import threading
from concurrent import futures
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
options = Options()
options.set_headless()
# options.add_arguments("--disable-notifications")
debug = True
duration = 4
url = 'https://telugu.pratilipi.com/read?id=6755373518909383'
# proverbs
url = 'https://telugu.pratilipi.com/read?id=6755373518926758'
def read_url(url):
driver = Firefox(options=options)
driver.maximize_window()
driver.get(url)
time.sleep(4)
def __init__(self, client="firefox", username="******", proxy=None, command_executor=None, loadstyles=False,
profile=None, headless=False, autoconnect=True, logger=None, extra_params=None, chrome_options=None):
"""Initialises the webdriver"""
self.logger = logger or self.logger
extra_params = extra_params or {}
if profile is not None:
self._profile_path = profile
self.logger.info("Checking for profile at %s" % self._profile_path)
if not os.path.exists(self._profile_path):
self.logger.critical("Could not find profile at %s" % profile)
raise WhatsAPIException("Could not find profile at %s" % profile)
else:
self._profile_path = None
self.client = client.lower()
if self.client == "firefox":
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
if not loadstyles:
# Disable CSS
self._profile.set_preference('permissions.default.stylesheet', 2)
# Disable images
self._profile.set_preference('permissions.default.image', 2)
# Disable Flash
self._profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
'false')
if proxy is not None:
self.set_proxy(proxy)
options = Options()
if headless:
options.set_headless()
options.profile = self._profile
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities['webStorageEnabled'] = True
self.logger.info("Starting webdriver")
self.driver = webdriver.Firefox(capabilities=capabilities, options=options, **extra_params)
elif self.client == "chrome":
self._profile = webdriver.ChromeOptions()
if self._profile_path is not None:
self._profile.add_argument("user-data-dir=%s" % self._profile_path)
if proxy is not None:
self._profile.add_argument('--proxy-server=%s' % proxy)
if headless:
self._profile.add_argument('headless')
if chrome_options is not None:
for option in chrome_options:
self._profile.add_argument(option)
self.logger.info("Starting webdriver")
self.driver = webdriver.Chrome(chrome_options=self._profile, **extra_params)
elif client == 'remote':
if self._profile_path is not None:
self._profile = webdriver.FirefoxProfile(self._profile_path)
else:
self._profile = webdriver.FirefoxProfile()
capabilities = DesiredCapabilities.FIREFOX.copy()
self.driver = webdriver.Remote(
command_executor=command_executor,
desired_capabilities=capabilities,
**extra_params
)
else:
self.logger.error("Invalid client: %s" % client)
self.username = username
self.wapi_functions = WapiJsWrapper(self.driver, self)
self.driver.set_script_timeout(500)
self.driver.implicitly_wait(10)
if autoconnect:
self.connect()
