def main():
args = parser.parse_args()
while (True):
# check if network is alive
if check_alive(url=args.website):
print('\n')
time.sleep(args.time_interval)
else: # network is not working
print('Lost Internet access. Trying to re-login ...')
if args.browser_driver is None:
browser = splinter.Browser(driver_name=args.browser,
headless=True)
else:
browser = splinter.Browser(driver_name=args.browser,
executable_path=args.browser_driver,
headless=True,
set_page_load_timeout=5)
try:
re_login(args, browser)
except RetryError:
print('Login failed.\n')
continue
finally:
browser.quit()
print('\n')
time.sleep(max(args.time_interval - 4, 0))
def create(self):
# handle like a switch case
if self.core.Config["driver"] == "firefox":
profile = FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "127.0.0.1")
profile.set_preference("network.proxy.http_port", "8008")
profile.set_preference("network.proxy.no_proxies_on", "")
profile.set_preference('webdriver_enable_native_events', True)
profile.update_preferences()
browser = splinter.Browser('firefox', firefox_profile=profile)
return browser
elif self.core.Config["driver"] == "chrome":
options = ChromeOptions()
options.add_arguments("--proxy-server=http://127.0.0.1:8008/")
browser = splinter.Browser(
'chrome',
executable_path=self.core.Config["chromedriver_path"],
options)
return browser
elif self.core.Config["driver"] == "phantomjs":
service_args = ('--proxy=127.0.0.1:8008', '--proxy-type=http',
'--ignore-ssl-errors=true')
browser = splinter.Browser('phantomjs',
self.core.Config["phantomjs_path"],
service_args=service_args)
return browser
def before_scenario(context, scenario):
browser_name = os.getenv('BROWSER', 'chrome')
if browser_name == 'chrome' and os.getenv('HEADLESS') == 'true':
browser = splinter.Browser('chrome', headless=True)
else:
browser = splinter.Browser(driver_name=browser_name)
# Initialize browser and add driver to global context
context.browser = browser
def setUpClass(cls):
kwargs = {}
if WEBDRIVER == 'phantomjs':
# XXX Might add '--remote-debugger-port=9000' but throws exception.
kwargs['service_args'] = [
'--webdriver-loglevel=DEBUG'] if DEBUG else []
try:
cls.browser = splinter.Browser(WEBDRIVER, **kwargs)
except splinter.exceptions.DriverNotFoundError:
cls.browser = splinter.Browser('firefox')
super(TestBrowser, cls).setUpClass()
def before_scenario(context, scenario):
""" Before each scenario, we need to restart the browser session so that
there's no bleed-over from previous tests.
"""
if settings.BROWSER == 'remote':
context.browser = splinter.Browser(
driver_name='remote',
browser='chrome',
url=settings.SELENIUM_URL,
)
else:
context.browser = splinter.Browser(
driver_name=settings.BROWSER,
headless=settings.RUN_HEADLESS,
)
def init_browser(webdriver='chrome', options=Options()):
if webdriver == 'chrome':
return splinter.Browser(
webdriver,
options=options,
service_log_path=os.path.devnull,
user_agent=
'Mozilla/5.0 ;Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; like Gecko'
)
else:
return splinter.Browser(
webdriver,
user_agent=
'Mozilla/5.0 ;Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; like Gecko'
) # headless=headless,
def launch_browser(self):
self.browser = splinter.Browser()
self.browser.visit('http://faculty.fso.fullsail.edu')
if self.login == True:
self.browser.find_by_id('username').fill(self.username)
self.browser.find_by_id('password').fill(self.pw)
self.browser.find_by_text('Login').click()
def scrape():
print(
"Starting Mars Scraping. Browser will be closed automatically after 4 parts are done."
)
# Setting Chrome Driver exe file path
executable_path = {"executable_path": "Resources/chromedriver.exe"}
browser = splinter.Browser("chrome", **executable_path)
news_title, news_p = mars_news(browser, "https://mars.nasa.gov/news/")
featured_image_url = featured_image(
browser, "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars")
mars_fact_table = get_mars_fact("https://space-facts.com/mars/")
#Close browser
browser.quit()
list_hemispheres = get_hemispheres(
"https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
)
# Gather all scraped data into a python dictionary
mars_data = {
"news_title": news_title,
"news_p": news_p,
"featured_img": featured_image_url,
"mars_fact": mars_fact_table,
"hemisphere_img": list_hemispheres
}
return mars_data
def webkit2png(url, image_file_path, wait_time=10):
with splinter.Browser('phantomjs') as browser:
browser.visit(url)
if browser.status_code.is_success():
try:
with wait_for_mathjax_loading(browser.driver):
pass
except TimeoutException:
pass
if wait_time > 0:
time.sleep(wait_time)
browser.driver.save_screenshot(image_file_path)
image = Image.open(image_file_path)
image.load()
if is_transparent(image) and False:
no_alpha = Image.new('L', image.size, (255))
no_alpha.paste(image, mask=image.split()[-1])
else:
no_alpha = image.convert('L')
# Convert to black and white imageage.
bw = no_alpha.convert('L')
# bw = bw.filter(ImageFilter.MedianFilter)
# White background.
bg = Image.new('L', image.size, 255)
bbox = ImageChops.difference(bw, bg).getbbox()
if bbox:
image = image.crop(bbox)
image.save(image_file_path)
def __init__(self, context):
self._user = context.demo_user
self._browser = browser = splinter.Browser()
self._cockpit_url = "http://localhost:9090"
self._plugin_frame = None
self._scenario_cleanup = cleanup = context.scenario_cleanup
cleanup.enter_context(browser)
def init_browser(webdriver="chrome", headless=True):
if webdriver == "chrome":
return splinter.Browser(
webdriver,
headless=headless,
service_log_path=os.path.devnull,
user_agent=
"Mozilla/5.0 ;Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; like Gecko"
)
else:
return splinter.Browser(
webdriver,
# headless=headless,
user_agent=
"Mozilla/5.0 ;Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; like Gecko"
)
def run(self):
with splinter.Browser('chrome',
executable_path=EXECUTABLE_PATH,
service_args=PROXIES) as browser:
for site in SITES:
browser.visit(site)
company_name = browser.find_by_css(
'#welcomeMessage > h1').value
menu_bar = browser.find_by_css(
'#menuBar > div.menuItemNoSub:last-child > a')
if menu_bar:
menu_bar.click()
self.__browse_all(site, company_name, browser)
else:
for menu_item_no_sub in browser.find_by_css(
'#tabHeader > ul > li > a'):
if menu_item_no_sub.value.strip() == 'By Location':
menu_item_no_sub.click()
browser.find_by_css(
'#oneClickLocation > div > a').click()
self.__browse_all(site, company_name, browser)
break
self.posting_id_queue.put(None)
def launch_browser(self):
self.browser = splinter.Browser()
self.browser.visit(self.url)
if self.login == True:
self.browser.find_by_id('username').fill(self.username)
self.browser.find_by_id('password').fill(self.pw)
self.browser.find_by_text('Login').click()
def main():
targ_dir = os.getcwd()
if len(sys.argv) == 1:
targ_lang, targ_file = decide_file(targ_dir)
elif len(sys.argv) == 2:
targ_lang, targ_file = decide_file(targ_dir, sys.argv[1])
else:
raise Exception("The number of input arguments is wrong")
contest_name, sub_name = os.getcwd().split('/')[-2:]
browser = splinter.Browser(BROWSER)
# sign in
signin_url = os.path.join(BASE_URL % contest_name, 'login')
browser.visit(signin_url)
browser.fill('name', USER_NAME)
browser.fill('password', PASSWORD)
browser.find_by_xpath(
'//*[@id="outer-inner"]/div[2]/div[1]/form/fieldset/div[3]/button'
).click()
# submit
submit_url = os.path.join(BASE_URL % contest_name, 'submit')
browser.visit(os.path.join(BASE_URL % contest_name, 'submit'))
f = open(targ_file, 'r')
browser.fill('source_code', ''.join(f.read()).decode('utf-8'))
f.close()
elements = browser.find_by_xpath('//*[@id="submit-task-selector"]/option')
for i in elements:
if i.text.startswith(sub_name):
i.click()
break
elements = browser.find_by_xpath(
'//select[@class="submit-language-selector"]')
targ_num = None
for num, i in enumerate(elements):
if i.visible:
targ_num = num + 1
break
elements = browser.find_by_xpath(
'//select[@class="submit-language-selector"][%d]/option' % targ_num)
dist = float('inf')
for i in elements:
i_dist = Levenshtein.distance(i.text, unicode(targ_lang))
if i_dist < dist:
dist = i_dist
i.click()
browser.find_by_xpath(
'//*[@id="outer-inner"]/div[2]/div/form/fieldset/div[4]/button/span/span[2]'
).click()
cur_status = browser.find_by_xpath(
'//*[@id="outer-inner"]/table/tbody/tr[1]/td[5]/span').text
while cur_status == 'WJ' or '/' in cur_status:
printr(cur_status)
time.sleep(5)
browser.reload()
cur_status = browser.find_by_xpath(
'//*[@id="outer-inner"]/table/tbody/tr[1]/td[5]/span').text
print cur_status + ' ' * 20
browser.quit()
def get_firefox_splinter(headless=True, proxy: str = None, **kwargs):
import splinter
config = {
'service_log_path': os.path.join(ostk.TEMPDIR, 'geckodriver.log'),
'headless': headless
}
config.update(kwargs)
profile_dict = {}
if proxy:
from urllib.parse import urlparse
prefix = 'network.proxy.'
profile_dict[prefix + 'type'] = 1
proxy_parse = urlparse(proxy)
scheme = proxy_parse.scheme
netloc = proxy_parse.netloc
try:
host, port = netloc.split(':')
port = int(port)
except ValueError:
raise ValueError(proxy)
if scheme in ('http', 'https', ''):
profile_dict[prefix + 'http'] = host
profile_dict[prefix + 'http_port'] = port
profile_dict[prefix + 'https'] = host
profile_dict[prefix + 'https_port'] = port
elif scheme.startswith('socks'):
profile_dict[prefix + 'socks'] = host
profile_dict[prefix + 'socks_port'] = port
else:
raise ValueError(proxy)
browser = splinter.Browser(driver_name='firefox',
profile_preferences=profile_dict,
**config)
return browser
def connect_splinter_proxy(count, url, proxyIP, proxyPort, wait_on_page_time):
wait_time = randrange(0.2*wait_on_page_time,1.8*wait_on_page_time)
print("Starting Browser#%s \t@ %s:%s to URL: %s \tWait time: %s sec"%(count,proxyIP,proxyPort,url,wait_time))
proxy_settings = {'network.proxy.type': 1,
'network.proxy.http': proxyIP,
'network.proxy.http_port': proxyPort,
'network.proxy.ssl': proxyIP,
'network.proxy.ssl_port':proxyPort,
'network.proxy.socks': proxyIP,
'network.proxy.socks_port':proxyPort,
'network.proxy.ftp': proxyIP,
'network.proxy.ftp_port':proxyPort
}
b= splinter.Browser(profile_preferences=proxy_settings)
b.driver.set_window_size(0, 0)
b.visit(url)
time.sleep(5)
if len(b.html)<20000:
b.visit(url)
time.sleep(5)
if len(b.html)<20000:
b.visit(url)
time.sleep(wait_time) ## 5 minutes to 20 minutes
b.quit()
print("\tClosing Browser#%s"%count)
return
def test_logout_redirect_to_index(self):
with splinter.Browser('django') as browser:
browser.visit(reverse('login'))
browser.visit(self.url)
self.assertEqual(browser.url, reverse('index'))
def get_event(offset: int) -> Tuple[datetime.date, str]:
"""
Scrape the event from meetup comments.
:param offset: Which event to get, i.e. 0 == this week, -1 last week etc.
:return: Tuple of the date of the event and the text of the comment
containing the teams.
"""
executable_path = DIR / '../bin/chromedriver'
with splinter.Browser('chrome',
executable_path=executable_path) as browser:
browser.visit(
"https://www.meetup.com/Ijburg-Futsal-Regulars/events/past/")
event_el = (browser.find_by_css('.eventCard--link') | to(list))[offset]
browser.visit(event_el['href'])
date = browser.find_by_tag('time')['datetime'][:-3] | to(
int) # strip milliseconds
for comment in browser.find_by_css('.comment-content'):
if 'Teams for' in comment.text:
event = comment.text
break
else:
raise ValueError('No teams comment found')
date = datetime.datetime.utcfromtimestamp(date).date()
return date, event
def __init__(self):
self.captcha_element = None
self.browser = splinter.Browser()
self.neural_net = nn.NeuralNetwork(weights_file=config['weights_path'])
self.num_correct = 0
self.num_guesses = 0
self.old_captcha_urls = []
def test_new_registered_user_send_an_email(self, csrf_mock):
self.assertEqual(len(mail.outbox), 0)
with splinter.Browser('django') as browser:
browser.visit(self.url)
self.assertEqual(auth.get_user_model().objects.count(), 0)
self.fill_and_send_form(
browser=browser,
username='******',
password1='fake_password',
password2='fake_password',
email='*****@*****.**',
)
self.assertEqual(auth.get_user_model().objects.count(), 1)
self.assertEqual(len(mail.outbox), 1)
email_sent = mail.outbox[0]
self.assertEqual(email_sent.subject, 'New user was registered')
self.assertEqual(email_sent.from_email, settings.APP_EMAIL)
self.assertTrue(len(email_sent.to), 1)
self.assertEqual(email_sent.to[0], settings.APP_EMAIL)
# Body
self.assertEqual(email_sent.body, 'fake_user')
def __init__(self, group_name, driver, login_email=None, password=None, delay=1):
self.group_name = group_name
self.login_email = login_email
self.password = password
self.delay = delay
self.br = splinter.Browser(driver)
def update_backers_db_from_queue(project_queue, fail_queue, process_id):
ff_profile = os.getcwd() + "\\lib\\quick_firefox"
browser = splinter.Browser('firefox', profile=ff_profile)
while True:
p = project_queue.get()
if p is None: # signal that there are no more projects
browser.quit()
project_queue.task_done()
break
url = p['url']
projectid = p['id']
try:
backers = get_backers_from_url(url=url,
max_wait=30,
browser=browser)
data = tablib.Dataset()
data.headers = ['projectid', 'userid', 'name', 'raw_location']
for backer in backers:
row = (projectid, backer['id'], backer['name'],
backer['raw_location'])
data.append(row)
db = db_connections.get_fungrosencrantz_schema(
schema='kickstarter')
db_connections.uploadOutputFile(data=data, db=db, table='backer')
del db
print
"Success: " + url
except:
fail_queue.put(url)
print
"Failed: " + url
project_queue.task_done()
def test_check_logon():
browser = splinter.Browser('chrome')
browser.visit('file:///C:/Users/Administrator/Desktop/%E7%89%9B%E7%89%9B%E5%9B%BE%E4%B9%A6%E9%A6%86%E6%8F%90%E7%A4%BA.html')
if '如果您的浏览器没有自动跳转，请点击这里'.decode('gbk') in browser.html:
print "found"
search_btn = browser.find_by_xpath('/html/body/table/tbody/tr[2]/td/div/a')
search_btn.click()
def download_document(document_title):
browser = splinter.Browser('chrome')
browser.visit('http://www.cnki.net/')
input_box = browser.find_by_id('txt_1_value1')
input_box.fill(document_title.decode('gbk'))
search_btn = browser.find_by_id('btnSearch')
search_btn.click()
# browser.find_by_css("body")
browser.driver.switch_to_frame("iframeResult")
first_link = browser.find_by_xpath('//*[@id="ctl00"]/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/a')
first_link.click()
# browser = splinter.Browser('chrome')
# browser.visit('http://www.cnki.net/KCMS/detail/detail.aspx?QueryID=0&CurRec=1&recid=&filename=JXCY201401007&dbname=CJFD2014&dbcode=CJFQ&pr=&urlid=&yx=&v=MTYyODdTN0RoMVQzcVRyV00xRnJDVVJMeWVaK1JxRnk3bFZiclBMelhJZDdHNEg5WE1ybzlGWTRSOGVYMUx1eFk=')
detail_window = browser.driver.window_handles[1]
browser.driver.close()
browser.driver.switch_to.window(detail_window)
pdf_link = browser.find_by_xpath('//*[@id="QK_nav"]/ul/li[2]/a')
pdf_link.click()
def setUp(self):
session = create_session(self.config_name, num_participants=2)
self.p1, self.p2 = session.get_participants()
#path = channel_utils.wait_page_path(
# session.pk, group_id_in_subsession=1, index_in_pages=self.wait_page_index)
#self.p1_client = ConnectingWSClient(path=path)
self.br = splinter.Browser('django')
def launch_browser(self):
if self.browser_type == 'firefox':
self.browser = splinter.Browser() #firefox
if self.browser_type == 'phantomjs':
self.browser = splinter.Browser('phantomjs') #phantomjs
self.browser.driver.set_window_size(1400, 800)
self.browser.visit(self.url)
if self.login == True:
self.browser.find_by_id('username').fill(self.username)
self.browser.find_by_id('password').fill(self.pw)
self.browser.find_by_text('Login').click()
else:
while True:
i = input('Please login to continue. \nPress Y to continue:'
).lower()
if i == 'y':
break
def get_headless_browser(browser_type='phantomjs', user_agent=USER_AGENT_FIREFOX_WIN10) -> splinter.Browser:
b = splinter.Browser(
browser_type,
user_agent=user_agent,
service_args=['--webdriver-loglevel=WARN'],
service_log_path=os.path.join(TEMPDIR, 'ghostdriver.log'),
)
b.driver.set_window_size(800, 600)
return b
def browser_chrome(context, timeout=30, **kwargs):
"""
Adds a browser to the current context. Adds a cleanup step to quit the browser
on end/fail
"""
if settings.BROWSER == 'remote':
browser = splinter.Browser(
driver_name='remote',
browser='chrome',
url=settings.SELENIUM_URL,
)
else:
browser = splinter.Browser(
driver_name=settings.BROWSER,
headless=settings.RUN_HEADLESS,
)
context.browser = browser
context.add_cleanup(browser.quit)
return browser
def before_scenario(context, scenario):
use_fixture(django_test_case, context)
context.base_url = context.test.live_server_url
os.environ['APP_URL'] = context.base_url
purge_old_screenshots(context)
browser = splinter.Browser('chrome', headless=True, executable_path=ChromeDriverManager().install())
context.browser = browser
def longxi_continuous(passwd, interval=300):
'''longxi到期后,自动重新认证'''
browser = splinter.Browser("firefox")
while True:
longxi_login_url = longxi_timeout_p("http://www.baidu.com")
if longxi_login_url:
browser.visit(longxi_login_url)
browser.fill()
time.sleep(interval)