def buyprd(u):
browser = Browser('firefox')
url = u
browser.visit(url)
# 10|10.5
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
print "checking out"
browser.visit(checkoutUrl)
print "Filling Out Billing Info"
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print "Filling Out Address"
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
print "Filling Out Credit Card Info"
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
print "Submitting Info"
browser.find_by_name('commit').click()
sys.exit(0)
def buyprd(u):
browser = Browser('firefox')
url = u
browser.visit(url)
# 10|10.5
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
print("checking out")
browser.visit(checkoutUrl)
print("Filling Out Billing Info")
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print("Filling Out Address")
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
browser.select("order[billing_country], country)
print("Filling Out Credit Card Info")
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
print("Submitting Info")
browser.find_by_name('commit').click()
sys.exit(0)
def hemisphere(browser):
executable_path = {"executable_path": "./chromedriver.exe"}
browser = Browser("chrome", **executable_path)
# Visit the USGS Astrogeology Science center site
url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
browser.visit(url)
hemisphere_image_urls = []
# Get a list of all the hemisphere
links = browser.find_by_css("a.product-item h3")
for item in range(len(links)):
hemisphere = {}
# Find element on each loop to avoid stale element exception
browser.find_by_css("a.product-item h3")[item].click()
# Find sample image anchor tag & extract href
sample_element = browser.find_link_by_text("Sample").first
hemisphere["img_url"] = sample_element["href"]
# Get hemisphere title
hemisphere["title"] = browser.find_by_css("h2.title").text
# Append hemisphere object to list
hemisphere_image_urls.append(hemisphere)
# Navigate back
browser.back()
return hemisphere_image_urls
def hemisphere_images(browser):
browser = Browser('chrome', 'chromedriver.exe', headless=False)
# Visit URL
url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser.visit(url)
hemisphere_image_urls = []
links = browser.find_by_css("a.product-item h3")
try:
for i in range(4):
hemisphere = {}
browser.find_by_css("a.product-item h3")[i].click()
sample_elem = browser.links.find_by_text('Sample').first
hemisphere['img_url'] = sample_elem['href']
hemisphere['title'] = browser.find_by_css("h2.title").text
hemisphere_image_urls.append(hemisphere)
browser.back()
except AttributeError:
return None
# browser.quit()
return hemisphere_image_urls
def mars_hemispheres():
url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser = Browser("chrome", executable_path="chromedriver", headless=True)
browser.visit(url)
hemisphere_image_urls = []
# First, get a list of all of the hemispheres
links = browser.find_by_css("a.product-item h3")
# Next, loop through those links, click the link, find the sample anchor, return the href
for i in range(len(links)):
hemisphere = {}
# We have to find the elements on each loop to avoid a stale element exception
browser.find_by_css("a.product-item h3")[i].click()
# Next, we find the Sample image anchor tag and extract the href
sample_elem = browser.links.find_by_text('Sample').first
hemisphere['img_url'] = sample_elem['href']
# Get Hemisphere title
hemisphere['title'] = browser.find_by_css("h2.title").text
# Append hemisphere object to list
hemisphere_image_urls.append(hemisphere)
# Finally, we navigate backwards
browser.back()
return hemisphere_image_urls
def get_featured_image():
print("--------------------------------")
print(" Getting Featured Image ")
print("--------------------------------")
# opening browser using splinter
executable_path = {'executable_path': 'C:/Users/swati/Downloads/chromedriver_win32/chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=False)
# URL of page to be scraped
url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
# opening the website
browser.visit(url)
# clicking on link to enlarge the image
browser.find_by_css('.fancybox').first.click()
# stop for a bit so the click changes can be persisted
# if you remove this there is a possibility that
# you will get html before click makes any changes
time.sleep(3)
# getting the html for the website
html=browser.html
soup = bs(html, 'lxml')
soup.prettify()
# find information for featured image
results = soup.find('img', class_="fancybox-image")
print(results)
# extracting the url
image_url = "https://www.jpl.nasa.gov" + results['src']
print(image_url)
browser.quit()
return image_url
def scrape_hemisphere_enhanced_images():
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=True)
base_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
all_hemispheres = []
browser.visit(base_url)
num_hemispheres = len(browser.find_by_css(".thumb"))
for hemisphere_num in range(num_hemispheres):
curr_title = browser.find_by_tag("h3")[hemisphere_num].html.replace(
" Enhanced", "")
browser.find_by_css(".thumb")[hemisphere_num].click()
curr_img_url = browser.find_by_text("Sample").first["href"]
# print(curr_img_url)
browser.back()
all_hemispheres.append({"title": curr_title, "img_url": curr_img_url})
browser.windows[0].close_others()
# print(all_hemispheres)
browser.quit()
return all_hemispheres
def main(aids):
livelogin = '******'
sboxlogin = '******'
livesearch = 'https://admin.ebizautos.com/index.cfm?section=accounts&page=manage&settings=overview&aid='
sboxsearch = 'https://admin.sandbox.ebizautos.com/index.cfm?section=accounts&page=manage&settings=overview&aid='
livecp = 'https://cp.ebizautos.com/index.cfm?ebxid=314'
sboxcp = 'https://cp.sandbox.bizautos.com/index.cfm?ebxid=314'
login = False
while login == False:
username = input('Username: '******'Password: '******'username', username)
browser.fill('Password', password)
try:
browser.find_by_xpath(
'/html/body/div[1]/div/div[1]/form/table/tbody/tr[6]/td/div[1]/a'
).click()
time.sleep(2)
for aid in aids:
browser.visit(livesearch + str(aid))
browser.find_by_css(
'html body#MainAdminBody div#BodyContainerWide div#MainBody.ClearFix div#LeftNav div.Content div div.Nav3 a'
).click()
browser.driver.find_element_by_tag_name('body').send_keys(
Keys.CONTROL, '1')
login = True
except:
print('Login error\a')
browser.quit()
def scrape_URL_5():
#Leverage Splinter to Web Scrape
executable_path = {'executable_path': r"/home/bdr/Desktop/chromedriver"}
browser = Browser('chrome', **executable_path, headless=True)
#*** URL 5: Mars Hemispheres ****
url = ("https://astrogeology.usgs.gov/search/"
"results?q=hemisphere+enhanced&k1=target&v1=Mars")
browser.visit(url)
#html_text = browser.html
#Empty dictionary to hold values
hemisphere_image_urls = []
for i in range(4):
# Find the elements on each loop to avoid a stale element exception
browser.find_by_css("a.product-item h3")[i].click()
hemi_data = scrape_hemisphere(browser.html)
# Append hemisphere object to list
hemisphere_image_urls.append(hemi_data)
# Finally, we navigate backwards
browser.back()
#Close Browser
browser.quit()
# Create Dictionary with Target Information
rtnDict = {"hemisphere_url_images": hemisphere_image_urls}
return rtnDict
def hemispheres():
url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
executable_path = {
'executable_path': 'chromedriver.exe'
} #path to chromedriver.exe; in this case it's in the same file
browser = Browser('chrome', **executable_path,
headless=False) #gives us a broswer object
browser.visit(url) #visit url
html_code = browser.html #get html text from page
soup = bs(html_code, "html.parser") #beautiful soup to parse
data = soup.find('div',
class_='collapsible results').find_all('div',
class_='item')
data_dict = []
for i in range(len(data)):
title = data[i].find('div', class_="description").find('h3').text
browser.find_by_css('div[class="collapsible results"]').find_by_css(
'div[class="item"]')[i].find_by_css(
'div[class="description"]').find_by_css('a').click()
for img in browser.find_by_css('div[class="downloads"]').find_by_css(
'a'):
if ('Sample' in img.text):
img_url = img['href']
browser.back()
mydict = {'title': title, 'img_url': img_url}
data_dict.append(mydict)
browser.quit()
return data_dict
def run(self):
browser = Browser(
'chrome',
executable_path=
"C:\Python27\Lib\site-packages\chromedriver_win32\chromedriver.exe",
service_args=PROXIES)
# browser = Browser('phantomjs', service_args=PROXIES, user_agent="Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0")
with browser:
page = 1
browser.visit(self.url)
browser.fill("p", self.keyword)
browser.find_by_id("search-submit").click()
while True:
time.sleep(10)
logging.info("Page " + str(page))
for link in browser.find_by_css("div.res"):
if "applicationname" in link.find_by_css(
"a").first["href"].lower():
self.queue.put(link.find_by_css("a").first["href"])
page += 1
if browser.find_by_css("#pg-next"):
browser.find_by_css("#pg-next").click()
else:
break
self.queue.put(None)
def hemisphere_images(browser):
# Setup splinter
# executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', 'chromedriver.exe', headless=False)
# Visit URL
url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser.visit(url)
hemisphere_image_urls = []
# html = browser.html
# image_soup = soup(html, 'html.parser')
links = browser.find_by_css("a.product-item h3")
try:
for i in range(len(links)):
hemisphere = {}
browser.find_by_css("a.product-item h3")[i].click()
sample_elem = browser.links.find_by_text('Sample').first
hemisphere['img_url'] = sample_elem['href']
hemisphere['title'] = browser.find_by_css("h2.title").text
hemisphere_image_urls.append(hemisphere)
browser.back()
except AttributeError:
return None
# browser.quit()
return hemisphere_image_urls
def match_subcrawler(match_url=None):
def trim_string(s):
s = s.rstrip()
s = s.lstrip()
return re.sub(" +", " ", s)
try:
browser = Browser("phantomjs")
browser.visit(match_url)
match = dict()
match["day"], match["date"], match["time"] = browser.find_by_css("p.datet").text.split(", ")
match["day"] = trim_string(match["day"])
match["date"] = trim_string(match["date"])
match["time"] = trim_string(match["time"])
participants = browser.find_by_id("col-content").find_by_tag("h1").text
if " - " not in participants:
error_print("match_subcrawler", "participant string malformat (miss ' - ')")
return None
match["team_H"], match["team_A"] = participants.split(" - ")
match["score"] = dict()
match["score"]["team_H"], match["score"]["team_A"] = browser.find_by_css("p.result").find_by_tag("strong").text.split(":")
odds_aver = browser.find_by_css("tr.aver").find_by_css("td.right")
if odds_aver.is_empty():
error_print("match_subcrawler", "can't find odds for" + participants + match["date"])
return match
match["odd_H"] = odds_aver[0].text
match["odd_D"] = odds_aver[1].text
match["odd_A"] = odds_aver[2].text
return match
except HttpResponseError, e:
error_print("match_subcrawler",
match_url + ": " + e.msg)
return None
def test_0_http_browser_download(self):
path = self.get_endpoint_path("HTTPServer")
url = "http://{0}/thredds/fileServer/{1}".format(self.data_node, path)
OpenID = "https://{0}/esgf-idp/openid/{1}".format(self.idp_node, self.username)
pf = {"browser.helperApps.neverAsk.saveToDisk": "application/x-netcdf, application/netcdf"}
browser = Browser("firefox", profile_preferences=pf)
browser.visit(url)
if browser.status_code.is_success() is True:
browser.quit()
return
browser.find_by_css("input.custom-combobox-input").fill(OpenID)
browser.find_by_value("GO").click()
browser.find_by_id("password").fill(self.password)
browser.find_by_value("SUBMIT").click()
# To Do only if user is not enrolled in a group
if browser.is_text_present("Group Registration Request"):
# Chosing First Registration Group
browser.find_by_id("button_1").click()
# Accepting License Agreement
browser.execute_script("myForm.submit();")
# Clicking on 'Download data button'
browser.find_by_id("goButton").click()
browser.quit()
def scrape_featured_mars_image():
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=True)
base_url = "https://www.jpl.nasa.gov"
image_url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
browser.visit(image_url)
browser.find_by_css('img.thumb').first.click()
time.sleep(2)
browser.execute_script(
"document.getElementById('fancybox-lock').scrollTo(0, document.body.scrollHeight);"
)
browser.links.find_by_partial_text("more info").click()
time.sleep(1)
#get image src
img_soup = bs(browser.html, "html.parser")
img_src = img_soup.find("img", class_="main_image")["src"]
img_src = base_url + img_src
browser.quit()
return {"featured_image": img_src}
def dosmth(*arg):
#First argument True or False. When True use proxy in file 'fname', else use machine ip.
print arg
if not arg[0]:
browser = Browser()
browser.visit("http://www.ip-ping.ru/")
print browser.find_by_css("div.hc2").text
browser.quit()
else:
fname = arg[1]
with open(fname, 'r') as f:
reader = csv.reader(f, delimiter=":")
profiles = []
for ip, port in reader:
#print type(ip), port
profiles.append("{'network.proxy.http':'" + ip +
"','network.proxy.http_port':" + port +
",'network.proxy.type': 1}")
for profile in profiles:
print profile
profile = ast.literal_eval(profile)
browser = Browser(profile_preferences=profile)
browser.visit("http://www.ip-ping.ru/")
print browser.find_by_css("div.hc2").text
browser.quit()
class LoginTestCase(unittest.TestCase):
"""
Proposed solution for task 3.2
Splinter
Firefox
"""
def setUp(self):
self.browser = Browser()
self.browser.visit("http://diabcontrol1.herokuapp.com")
def tearDown(self):
self.browser.quit()
def _login(self, username, password):
self.browser.fill('username', username)
self.browser.fill('password', password + Keys.RETURN)
def test_login_success(self):
self._login('admin', 'admin123')
page_header = self.browser.find_by_css('.container-fluid .page-header')
self.assertIn('Welcome to DiabControl system', page_header.text)
def test_login_failed(self):
self._login('admin', 'INVALIDpassword')
alert = self.browser.find_by_css('.container-fluid .form .alert')
expected_warning = ('Please enter a correct username and password. '
'Note that both fields may be case-sensitive.')
self.assertIn(expected_warning, alert.text)
def main():
iphoneSix = {"deviceName": "iPhone 6"}
chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_experimental_option("mobileEmulation", iphoneSix)
chromeBrowser = Browser(CHROME, options=chromeOptions)
chromeBrowser.visit(STOREFRONT_URL_PRODUCTION + AU_PATH)
chromeBrowser.find_by_css('li[class=ht-search-li').click()
for key in chromeBrowser.type('query',
'car mats' + Keys.ENTER,
slowly=True):
pass
# scroll down then scroll up OPEN
assert chromeBrowser.find_by_id('productSelect')
chromeBrowser.find_by_id('ht-sbc-tab2').click()
for key in chromeBrowser.type('make-input', 'mazda', slowly=True):
pass
chromeBrowser.find_by_css('li[rel="34"').click()
for key in chromeBrowser.type('model-input', 'mazda 6', slowly=True):
pass
chromeBrowser.find_by_css('li[rel="243"').click()
chromeBrowser.find_by_css('li[rel="2"').click() #Wagon
chromeBrowser.find_by_css('li[rel="129"').click() #2012-Current
chromeBrowser.find_by_id('vehicleSearchCategoryButton').click()
time.sleep(5)
chromeBrowser.quit()
def weibo():
web_firefox = Browser('chrome')
web_firefox.visit('http://weibo.com/login.php')
if web_firefox.find_by_name('username'):
print 'uname'
# web_firefox.find_by_name('username').click()
# 这个地方用by_name能找到，但是模拟点击时有问题，改成by_css
web_firefox.find_by_css("input[node-type=\"username\"]").fill(
'*****@*****.**')
time.sleep(random.randint(3, 10))
if web_firefox.find_by_name('password'):
print 'passwd'
web_firefox.find_by_css("input[node-type=\"password\"]").fill('******')
time.sleep(random.randint(3, 15))
print web_firefox.find_by_css(
".loginbox .W_login_form .login_btn div a ")[0].click()
time.sleep(random.randint(3, 10))
print web_firefox.find_by_css(".input .W_input").fill(
u'我说不让你用微博测试，你非用，得了吧。。。。 封ip了吧')
time.sleep(random.randint(3, 10))
print web_firefox.find_by_css("a[node-type=\"submit\"]").click()
def member_new(emails,link,password):
browser=Browser('chrome')
for n in emails:
email=n.rstrip()+"@sgsaas.net"
browser.visit(link);
browser.fill('age',"33");
browser.fill('firstname',"Sam");
browser.fill('lastname',"wang");
browser.find_by_name('email').last.fill(email);
browser.fill('telephone',3984020);
browser.fill('address_1',"Bukit Batok Avenue")
browser.fill('city',"Singapore");
browser.fill('postcode',658080);
browser.find_by_xpath('//select[@id="input-country"]//option[@value="188"]').click();
browser.find_by_xpath('//select[@id="input-zone"]//option[@value="4231"]').click();
browser.find_by_name('password').last.fill(password);
browser.fill('confirm',password);
browser.find_by_name('agree').click();
browser.find_by_css('.btn-primary').click();
sleep(1)
print email+" has registered!"
browser.quit()
return;
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = User(name="Alice",
email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run,
kwargs={"port": 8080})
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_2_add_post(self):
self.browser.visit("http://127.0.0.1:8080")
print("current url = ", self.browser.url)
self.browser.driver.set_window_size(1920, 1080)
self.browser.click_link_by_text('login')
print("current url = ", self.browser.url)
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
print(self.browser.url)
add_link = self.browser.find_link_by_partial_text('add')
add_link.click()
print(self.browser.url)
title = "test_acceptance_add_post"
self.browser.fill("title", title)
now = datetime.datetime.now()
now = str(now)
self.browser.fill("content", now)
button = self.browser.find_by_css("button[type=submit]")
button.click()
print(self.browser.url)
new_post_appears = self.browser.is_text_present(
title) and self.browser.is_text_present(now)
print("new_post_appears = ", new_post_appears)
self.assertEqual(new_post_appears, True)
def post_message(create_msg_url, category, title, message, send):
"""
Create a new U-Cursos forum's message (a.k.a. thread).
:param category: Specific for community message category
:param create_msg_url: URL of new message view (syntax is https://www.u-cursos.cl/<institution>/<year>/<semester>/<community>/<section>/foro/mensaje)
:param title: Message Title
:param message: Message content
:param send: Boolean flag to send the message to the forum
:return:
"""
# config
browser = Browser('chrome')
browser.visit(create_msg_url)
# log in
browser.fill(USERNAME_FIELD_NAME, private_credentials.username)
browser.fill(PASSWORD_FIELD_NAME, private_credentials.password)
browser.find_by_value(LOG_IN_BUTTON_VALUE).click()
# build up message
# choose category
browser.find_by_id(CATEGORY_CHOSER_ID).first.click()
browser.find_by_css(CATEGORY_CHOSER_TEXT_INPUT_CSS_CLASS).first.fill(
"{}\n".format(category))
# write title
browser.find_by_name(TITLE_FIELD_NAME).fill(title)
# write message
browser.find_by_name(MESSAGE_INPUT_NAME).fill(message)
# send message (CAUTION)
if send:
browser.find_by_value(SEND_BUTTON_VALUE).click()
def loginweibo():
web_firefox = Browser('chrome')
web_firefox.visit('http://weibo.com/login.php')
# button = browser.find_by_xpath("//input[@value='Sign In']")
if web_firefox.find_by_name('username'):
print 'uname'
# web_firefox.find_by_name('username').click()
# 这个地方用by_name能找到，但是模拟点击时有问题，改成by_csshttp://www.djhull.com/python/training-python-5.html
web_firefox.find_by_css("input[node-type=\"username\"]").fill('xxx')
time.sleep(random.randint(3, 10))
if web_firefox.find_by_name('password'):
print 'passwd'
web_firefox.find_by_css("input[node-type=\"password\"]").fill('xxx')
time.sleep(random.randint(3, 15))
# print web_firefox.find_by_css(".loginbox .W_login_form .login_btn a ")[0].click()
print web_firefox.find_by_css("span[node-type=\"submitStates\"]").click()
time.sleep(random.randint(3, 10))
print web_firefox.find_by_css(".input .W_input").fill(
u'我说不让你用微博测试，你非用，得了吧。。。。 封ip了吧')
time.sleep(random.randint(3, 10))
print web_firefox.find_by_css("a[node-type=\"submit\"]").click()
def getEstado():
browser = Browser('chrome')
browser.visit('http://scw.pjn.gov.ar/scw/home.seam')
candadito = browser.find_by_tag('li')[3]
candadito.click()
browser.fill('username', 'USERNAME')
browser.fill('password', 'PASSWORD')
browser.find_by_id("kc-login").click()
browser.find_by_css('.fa-list-ul').click()
browser.find_by_tag('li')[4].click()
for i in nroExptes:
try:
browser.find_by_css('.fa-share').click()
time.sleep(2)
browser.find_by_css('.form-control')[1].fill(i['nro'])
browser.find_by_value('Consultar').click()
time.sleep(2)
numero = browser.find_by_css('.column')[0].value
juzgado = browser.find_by_css('.column')[1].value
autos = browser.find_by_css('.column')[2].value
estado = browser.find_by_css('.column')[3].value
mensaje = autos + ": " + estado
print mensaje
resultado.append(mensaje)
except Exception as e:
print "Error con expte " + i['nro']
browser.quit()
enviarMail()
def dosmth(*arg):
#First argument True or False. When True use proxy in file 'fname', else use machine ip.
print arg
if not arg[0]:
browser = Browser()
browser.visit("http://www.ip-ping.ru/")
print browser.find_by_css("div.hc2").text
browser.quit()
else:
fname = arg[1]
with open(fname, 'r') as f:
reader = csv.reader(f, delimiter=":")
profiles = []
for ip, port in reader:
#print type(ip), port
profiles.append("{'network.proxy.http':'"+ip+"','network.proxy.http_port':"+port+",'network.proxy.type': 1}")
for profile in profiles:
print profile
profile = ast.literal_eval(profile)
browser = Browser(profile_preferences = profile)
browser.visit("http://www.ip-ping.ru/")
print browser.find_by_css("div.hc2").text
browser.quit()
def hemisphere():
# Visit the USGS Astrogeology Science Center Site
executable_path = {"executable_path": "/usr/local/bin/chromedriver"}
browser = Browser("chrome", **executable_path, headless=False)
url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
browser.visit(url)
hemi_img_urls = []
# Get a List of All the Hemispheres
links = browser.find_by_css("a.product-item h3")
for item in range(len(links)):
hemisphere = {}
browser.find_by_css("a.product-item h3")[item].click()
# Find Sample Image Anchor Tag & Extract <href>
sample = browser.find_link_by_text("Sample").first
hemisphere["img_url"] = sample["href"]
# Get Hemisphere Title
hemisphere["title"] = browser.find_by_css("h2.title").text
# Append List
hemi_img_urls.append(hemisphere)
# Navigate Backwards
browser.back()
hemi_img_urls
return {'Hemisphere Image URLS': hemi_img_urls}
def google_login(user_name, password, code):
browser = Browser('firefox')
url = 'https://accounts.google.com/ServiceLogin'
browser.visit(url)
browser.find_by_id('Email').fill(user_name)
browser.find_by_id('next').click()
browser.find_by_id('Passwd').fill(password)
browser.find_by_id('signIn').click()
url1 = 'https://play.google.com/store?hl=jp'
browser.visit(url1)
browser.find_by_css('button.id-no-menu-change').click()
time.sleep(1)
browser.find_by_css('input.redeem-input-text-box').fill(code)
browser.find_by_id('id-redeem-ok-button').click()
time.sleep(2)
result = browser.find_by_css('div.redeem-invalid-code-msg').value
browser.quit()
return result
def run(self):
browser = Browser('chrome',
executable_path=EXECUTABLE_PATH,
service_args=PROXIES)
# browser = Browser('phantomjs', service_args=PROXIES, user_agent=USER_AGENT)
with browser:
page = 1
browser.visit(URL)
browser.fill("q", KEYWORD)
browser.find_by_name("btnG").click()
time.sleep(5)
while True:
time.sleep(10)
logging.info("Page " + str(page))
for link in browser.find_by_css("h3.r"):
if "applicationname" in link.find_by_css(
"a").first["href"].lower():
self.url_queue.put(link.find_by_css("a").first["href"])
page += 1
if browser.find_by_css("#pnnext"):
browser.find_by_css("#pnnext").click()
else:
break
self.url_queue.put(None)
def mars_facts():
executable_path = {'executable_path': 'chromedriver.exe'}
browser = Browser('chrome', **executable_path, headless=True)
url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser.visit(url)
# HTML object
html = browser.html
# Parse HTML with Beautiful Soup
soup = BeautifulSoup(html, 'html.parser')
hemispheres = []
for i in range(4):
browser.find_by_css('a.product-item h3')[i].click()
# HTML object
html = browser.html
# Parse HTML with Beautiful Soup
soup = BeautifulSoup(html, 'html.parser')
title = soup.find('h2', class_="title").get_text()
link = soup.find("a", text="Sample").get("href")
hemisphere = {"title": title, "link": link}
hemispheres.append(hemisphere)
browser.back()
pprint.pprint(hemisphere)
class Submitter:
def __init__(self, url, username, password, course_id, homework_id,
submit_list):
self._callback = None
self._browser = Browser()
self._url = url
self._username = username
self._password = password
self._course_id = course_id
self._homework_id = homework_id
self._submit_list = submit_list
def _login(self):
self._browser.visit(self._url)
self._browser.fill("i_user", self._username)
self._browser.fill("i_pass", self._password)
self._browser.find_by_id("loginButtonId").click()
def _nvi2course(self):
self._browser.find_link_by_partial_text(self._course_id).first.click()
self._browser.windows.current.close()
def _nvi2homework(self):
self._browser.find_link_by_partial_text("课程作业").first.click()
self._browser.find_link_by_partial_text(
self._homework_id).first.click()
def _submit(self, stu_id, grade, comment, ex_file):
xpath_str = '//tbody/tr[td[3]=' + stu_id + ']/td[last()]/a'
self._browser.find_by_xpath(xpath_str).last.click()
self._browser.fill('cj', grade)
self._browser.fill('pynr', comment)
if os.path.splitext(ex_file)[1] == '.pdf':
self._browser.driver.find_element_by_name('fileupload').send_keys(
ex_file)
submit_btn_css = 'div[class="sub-back sub-back-3 absolute"] > input[class="btn"]'
self._browser.find_by_css(submit_btn_css).first.click()
while not self._browser.is_text_present('关闭', wait_time=1):
pass
self._browser.find_by_text('关闭').click()
self._browser.back()
self._browser.back()
def add_single_task_callback(self, callback):
self._callback = callback
def start(self):
self._login()
self._nvi2course()
self._nvi2homework()
for stu_id, grade, comment, ex_file in self._submit_list:
self._submit(stu_id, grade, comment, ex_file)
self._callback([stu_id, grade, comment, ex_file])
self._browser.quit()
@staticmethod
def clean():
work_dir = os.getcwd()
os.remove(work_dir + "/geckodriver.log")
def test_login(self):
with pyvirtualdisplay.Display():
browser = Browser()
browser.visit("http://ui/accounts/login/")
browser.fill("login", "testuser")
browser.fill("password", "password")
browser.find_by_css(".btn-primary").click()
self.assertTrue(browser.find_by_text("Successfully signed in as testuser."))
def test_login(self):
with pyvirtualdisplay.Display():
browser = Browser()
browser.visit("http://ui:8080/accounts/login/")
browser.fill("login", "testuser")
browser.fill("password", "password")
browser.find_by_css(".btn-primary").click()
self.assertTrue(browser.find_by_text("Successfully signed in as testuser."))
def donghang(url,flight):
browser = Browser('chrome') # defaults to firefox
browser.visit(url)
# browser.visit('http://www.ceair.com/booking/sha-adnh-200724_CNY.html')
# browser.find_by_css('.ceair-poptip').click()
time.sleep(5)
browser.find_by_css('.ceair-input__inner_homesearch').first().fill('敦煌')
def __scrape(self, landing_page):
browser = Browser('chrome', executable_path='C:\Python27\Lib\site-packages\chromedriver_win32\chromedriver.exe', service_args=PROXIES)
# browser = Browser('phantomjs', service_args=PROXIES, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0')
with browser:
template1 = True
browser.visit(landing_page)
time.sleep(2)
nav = [x for x in browser.find_by_css('a.nav') if (x.text == 'Jobs by Location' or x.text == 'By Location')]
if len(nav) > 0:
nav[0].click()
else:
template1 = False
link = browser.url
state_index = 1
city_index = 1
while True:
browser.visit(link)
if not template1:
nav = browser.find_by_css('#tabHeader')
nav = nav.find_by_css('a')
nav[1].click()
states = browser.find_by_name('search.stateList.value')
state_list = states.find_by_tag('option')
print state_list[state_index].text
state_list[state_index].click()
if state_list[state_index].text != 'choose one...':
element = 'cityList_' + state_list[state_index].text
cities = browser.find_by_id(element)
city_list = cities.find_by_tag('option')
city_list[city_index].click()
if city_list[city_index].text != 'choose one...':
print city_list[city_index].text, state_list[state_index].text
browser.find_by_id('cityStateSearch').click()
links = None
try:
links = browser.find_by_css('a.withBubble')
except:
pass
if len(links) > 0:
for i in links:
b = Browser('chrome', executable_path='C:\Python27\Lib\site-packages\chromedriver_win32\chromedriver.exe', service_args=PROXIES)
# b = Browser('phantomjs', service_args=PROXIES, user_agent='Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0')
with b:
b.visit(i['href'])
self.__navigate_pages(b)
else:
self.__navigate_pages(browser)
city_index += 1
if city_index == len(city_list):
city_index = 0
state_index += 1
if state_index == len(state_list):
break
else:
state_index += 1
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = User(name="Alice",
email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run,
kwargs={"port": 8080})
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_login_correct(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")
def test_login_incorrect(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")
def test_add_entry(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.browser.click_link_by_text("Add Entry")
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/entry/add")
self.browser.fill("title", "Test Title")
self.browser.fill("content", "Test Content")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run, kwargs={"port": 8080})
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_2_add_post (self):
self.browser.visit("http://127.0.0.1:8080")
print ("current url = ", self.browser.url)
self.browser.driver.set_window_size(1920, 1080)
self.browser.click_link_by_text('login')
print ("current url = ", self.browser.url)
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
print (self.browser.url)
add_link=self.browser.find_link_by_partial_text('add')
add_link.click()
print (self.browser.url)
title="test_acceptance_add_post"
self.browser.fill("title", title)
now=datetime.datetime.now()
now=str(now)
self.browser.fill("content", now)
button = self.browser.find_by_css("button[type=submit]")
button.click()
print(self.browser.url)
new_post_appears=self.browser.is_text_present(title) and self.browser.is_text_present(now)
print ("new_post_appears = ", new_post_appears)
self.assertEqual(new_post_appears, True)
class TestViews(unittest.TestCase):
def setUp(self):
# Test setup
self.browser = Browser("phantomjs")
#setup the tables in the database
Base.metadata.create_all(engine)
#create an example user
self.user = User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
self.entry = Entry(title="test entry", content="test content")
session.add(self.entry)
session.commit()
self.process = multiprocessing.Process(target=app.run)
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
#Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_login_correct(self):
self.browser.visit("http://127.0.0.1:5000/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
#print(self.browser.url)
#print('asdf')
time.sleep(1)
self.assertEqual(self.browser.url, "http://127.0.0.1:5000/")
def test_login_incorrect(self):
self.browser.visit("http://127.0.0.1:5000/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
time.sleep(1)
self.assertEqual(self.browser.url, "http://127.0.0.1:5000/login")
#my acceptance test
def test_edit_entry_requires_login(self):
self.browser.visit("http://127.0.0.1:5000/entry/1/edit")
time.sleep(1)
self.assertEqual(self.browser.url, "http://127.0.0.1:5000/login?next=%2Fentry%2F1%2Fedit")
def get_claim_code_from_server():
browser = Browser('phantomjs', service_args=['--ignore-ssl-errors=true'])
browser.visit(ROOT_ADDRESS + "/merchant-login")
browser.fill_form({"email": USER_NAME, "password": PASSWORD})
browser.find_by_id("loginButton")[0].click()
time.sleep(5)
browser.visit(ROOT_ADDRESS + "/api-tokens")
browser.find_by_css(".token-access-new-button").find_by_css(".btn").find_by_css(".icon-plus")[0].click()
browser.find_by_id("token-new-form").find_by_css(".btn")[0].click()
return browser.find_by_css(".token-claimcode")[0].html
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = models.User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run)
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_login_correct(self):
# self.browser.visit("http://0.0.0.0:8080/login")
self.browser.visit(DEVSERVERURL+"/login")
# print self.browser.html
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, DEVSERVERURL+"/")
def test_login_incorrect(self):
self.browser.visit(DEVSERVERURL+"/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, DEVSERVERURL+"/login")
def test_add_user(self):
self.browser.visit(DEVSERVERURL+"/add_user")
self.browser.fill("name", "dude")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.browser.visit(DEVSERVERURL+"/post/add")
self.assertEqual(self.browser.url, DEVSERVERURL+"/post/add")
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = models.User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run)
self.process.start()
time.sleep(1)
def test_add_post(self):
log= logging.getLogger("unittest.TestCase")
################################## Login as Alice
#self.browser.visit("http://0.0.0.0:8080/login") # original line
self.browser.visit("http://127.0.0.1:5000/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
#self.assertEqual(self.browser.url, "http://0.0.0.0:8080/") # original line
# self.assertEqual(self.browser.url, "http://127.0.0.1:5000/") # ask sam about this line
############################################ add a test post #####################
self.browser.visit("http://127.0.0.1:5000")
self.browser.click_link_by_partial_href('add')
self.browser.fill("title", "post test1 title")
self.browser.fill("content", "post test1 content")
button = self.browser.find_by_css("button[type=submit]")
button.click()
post_found = self.browser.find_by_tag('h1').value #cheated here - made template title h2. how do we access? index?
#post_found = self.browser.find_by_text('post test1 title').value - didnt work
log.debug( "FIRSTH1= %r", post_found )
self.assertEqual(post_found, "post test1 title")
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def paginate(self, url):
#browser = Browser("chrome")
browser = Browser("phantomjs")
browser.visit(url)
pages = [browser.html]
for i in range(10):
try:
browser.find_by_css(".PagerLinks > a")[-2].click()
pages.append(browser.html)
except:
break
return pages
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example
self.user = models.User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run)
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def login(self, email, password):
self.browser.visit(TEST_SITE + "login")
self.browser.fill("email", email)
self.browser.fill("password", password)
button = self.browser.find_by_css("button[type=submit]")
button.click()
def test_login_correct(self):
self.login("*****@*****.**", "test")
self.assertEqual(self.browser.url, TEST_SITE)
def test_login_incorrect(self):
self.login("*****@*****.**", "test")
self.assertEqual(self.browser.url, TEST_SITE + "login")
def test_add_post(self):
self.login ("*****@*****.**", "test")
self.browser.visit(TEST_SITE + "post/add")
self.browser.fill("title", "acceptance test post 1")
self.browser.fill("content", "this is the content for the test case")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, TEST_SITE)
self.assertTrue(
self.browser.is_text_present("acceptance test post 1"))
class TestViews(unittest.TestCase):
def setUp(self):
""" Test Setup """
self.browser = Browser("phantomjs")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
# multiprocessing module gives you the ability to start and run other code simultaneously with your own scripts
# also allows you to communicate and control this code, by called methods such as start and terminate
# also provides features for implements concurrency in your applications
# in this test, you can't call app.run method as usual because this method is blocking and will stop the tests from running
# instead, you target which function to run
self.process = multiprocessing.Process(target=app.run, kwargs={"port": 8080})
self.process.start()
# time.sleep(1) in order to pause for a second to allow server to start
time.sleep(1)
def test_login_correct(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")
def test_login_incorrect(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")
def tearDown(self):
""" Test Teardown """
# Remove the tables and their data from the database
# Kill the server
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
# Exit the browser
self.browser.quit()
def pages():
page_list = []
browser = Browser()
url = "https://web.archive.org/web/20141216192721/http://andrewsmedina.com/"
browser.visit(url)
dirs = browser.find_by_css(".directory li a")
for d in dirs:
d.click()
posts = browser.find_by_css("#posts h2 a")
for post in posts:
page_list.append(post)
return page_list
def paginate(self, url):
browser = Browser("phantomjs")
browser.visit(url)
pages, page = [browser.html], 0
while browser.find_by_css(".pagingNext"):
if page > 10: break
page = page + 1
try:
browser.find_by_css(".pagingNext").first.click()
pages.append(browser.html)
except:
break
return pages
class TestViews(unittest.TestCase):
def setUp(self):
"""Test setup"""
#create an instance of the splinter.Browser class, using the (PhantomJS driver) <-- use for our automation
self.browser = Browser("phantomjs")
#set up the tables in the database
Base.metadata.create_all(engine)
#create an example user
self.user = models.User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
"""
Understand multiprocessing from lesson, and more
"""
#use multiprocessing module in order to start the Flask test server running
#since we need a server up to run our app, since our test will be visiting the actual site
self.process = multiprocessing.Process(target=app.run)
self.process.start()
time.sleep(1)
def tearDown(self):
"""Test teardown"""
#Remove tables and data from the database
self.process.terminate()
Base.metadata.drop_all(engine)
self.browser.quit()
def testLoginCorrect(self):
self.browser.visit("http://0.0.0.0:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://0.0.0.0:8080/")
def testLoginIncorrect(self):
self.browser.visit("http://0.0.0.0:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://0.0.0.0:8080/login")
if __name__ == "__main__":
unittest.main()
def main():
# Open new firefox browser window, login to indeed.com
br = Browser('firefox')
br.visit(LOGIN_URL)
br.fill_form({'email':INDEED_USERNAME,'password':INDEED_PASSWORD})
br.find_by_css('.input_submit').first.click()
resume_links = []
resume_index = 0
while True:
br.visit(RESUMES_URL + '&start=%d' % resume_index)
links = br.find_by_css('.app_link')
if len(links) > 0:
for link in links:
# Rewrite this. The Browser object cannot find the href value for some reason...
resume_links.append('http://www.indeed.com' + re.search(r'href=[\'"]?([^\'" >]+)', link.outer_html).group(1))
resume_index += 50
else:
resume_index = 1
break
print '[+] Resume Links Found: %d' % len(resume_links)
# Load the 1,000 most commonly used words into common_words
sys.stdout.write('[+] Loading Common Word List [ ]')
common_words = gen_comword_list()
sys.stdout.flush()
sys.stdout.write('\r[+] Loading Common Word List [DONE]\n')
for resume in resume_links:
br.visit(resume)
try:
#print '[+] Scanning %s\'s Resume...' % br.find_by_id("resume-contact").text
resume_words = parse_resume_words(br.html, common_words)
#print '\tUnique Words Found: %d' % len(resume_words)
for word in resume_words:
if not word in KEYWORDS:
KEYWORDS[word] = 1
else:
KEYWORDS[word] += 1
except:
pass
sys.stdout.write('[+] Sanning Resumes: {0:.0f}%\r'.format(float(resume_index)/len(resume_links) * 100))
sys.stdout.flush()
resume_index += 1
print '\n[+] Total Unique Keywords Found: %d' % len(KEYWORDS.keys())
print '============\nTOP 20 WORDS\n============'
for pair in sorted(KEYWORDS.items(), key=operator.itemgetter(1), reverse=True)[:20]:
print '%s: %d' % (pair[0], pair[1])
def twitter_login(username, password):
"""
Log in to Twitter and returns browser object
"""
browser = Browser()
# Login
browser.visit("https://www.twitter.com/")
browser.find_by_id("signin-email").first.value = username
browser.find_by_id("signin-password").first.value = password
browser.find_by_css(".js-submit").first.click()
return browser
def passwd(self):
if len(self.login) < 1 or len(self.panic) < 1 or len(self.user) < 1:
return false
b = Browser()
b.driver.set_window_size(900,900)
try:
b.visit("https://accounts.google.com/ServiceLogin?service=accountsettings")
b.fill('Email',self.user)
btn = b.find_by_id("next")
btn.click()
b.fill('Passwd',self.login)
btn = poll_for("#signIn", b)
btn.click()
b.visit("https://myaccount.google.com/security#signin")
btn = b.find_by_css(".vkq40d").first
if not btn == None:
print "not none"
btn.click()
poll_fill('Email',self.user, b)
btn = b.find_by_id("next")
btn.click()
poll_fill('Passwd',self.login, b)
btn = b.find_by_id("signIn")
btn.click()
time.sleep(INV)
btn = poll_for(".TCRTM", b)
btn.click()
poll_fill('Passwd',self.login, b)
btn = b.find_by_id("signIn")
btn.click()
p = poll_for(".Hj", b)
p.fill(self.panic)
p = b.find_by_css(".Hj")[1]
p.fill(self.panic)
btn = b.find_by_css(".Ya")
btn.click()
time.sleep(INV*5)
b.quit()
except:
traceback.print_exc(file=sys.stdout)
raw_input("Something went wrong...")
b.quit()
if self.attempts < self.max_attempts:
self.attempts += 1
self.passwd()
def qd():
## browser = Browser('firefox'
## , profile='D:\UsersDocuments\FFmissions')
browser = Browser('chrome')
browser.visit(juchaozhi)
time.sleep(1)
browser.find_by_name('username').fill(username)
browser.find_by_name('password').fill(password)
browser.find_by_name('button').click()
time.sleep(1)
if browser.is_text_present(username, wait_time=1):
print ('login success!')
logging.info('login success!')
browser.visit('http://best.pconline.com.cn/qiandao.html')
time.sleep(1)
## 添加今天是否有签到的判断，主要判断口id="JBar"，或者判断CSS中的元素，如.signStatus,.isgnUps
if browser.find_by_css('.signStatus').visible:
print ('you had signed!')
logging.info('you had signed!')
browser.quit()
else:
browser.find_by_xpath('//*[@id="JSignBox"]/a').click()
time.sleep(1)
print ('mission success!')
logging.info('mission success!')
browser.quit()
else:
print ('login fail!!!')
logging.info('login fail!!!')
def __authorize(self):
flow = dropbox.client.DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
b = Browser("chrome")
b.visit(authorize_url)
if not b.find_by_name("allow_access"):
b.find_by_name("login_email")[1].fill("*****@*****.**")
b.find_by_name("login_password")[1].fill("dharit1250")
b.find_by_css(".login-button")[0].click()
time.sleep(2)
b.find_by_name("allow_access").first.click()
code = b.find_by_id("auth-code").first.text
access_token, user_id = flow.finish(code)
self.client = dropbox.client.DropboxClient(access_token)
b.quit()
class Crawler(object):
def __init__(self):
self.browser = Browser()
def exists_element_with_css(self, css):
element = self.browser.find.by_css(css)
return True if element else False
def next_page(self):
number_results = self.browser.find_by_css(
'#breadcrumb > li:nth-child(5) > div > span')
number_results = number_results.replace('"', '')
number_results = number_results.replace(')', '')
number_results = number_results.replace('(', '')
number_results = number_results.replace('resultados', '')
number_results = int(number_results)
return number_results
def visit_href(self, number):
number = number_results / 6
i = 0
while i <= number:
self.browser.visit(
'http://kekanto.com.br/newsearch/page:%s?&cidade_id=266911&cat=36') % i
def write_txt(self):
if self.exists_element_with_css(
'#bizes-content-list > li.box-white.biz-card.js_biz_info'):
import pdb; pdb.set_trace()
for element in self.browser.find_by_css('#bizes-content-list > li.box-white.biz-card.js_biz_info'):
arquivo.write(str(element.text))
arquivo.write('/n')
arquivo.close()
def run(self):
self.browser.visit('http://kekanto.com.br/newsearch/page:1?&cidade_id=266911&cat=36')
number = self.next_page() / 6
i = 1
while i <= number:
self.write_txt()
i+=1
self.browser.visit('http://kekanto.com.br/newsearch/page:%s?&cidade_id=266911&cat=36') % i
def crawl(usr, pswd, out_path, driver="firefox"):
bsr = Browser(driver)
bsr.visit(URL_LOGIN)
bsr.find_by_id("phone1").fill(usr)
bsr.find_by_id("pswd").fill(pswd)
bsr.find_by_id("login").click()
if bsr.is_element_present_by_css("span.fw1.fs0.acnt"):
print "Successfully login!"
else:
print "Login failed, bye!"
bsr.visit("http://123.163.com/webmail/main/#mid=7")
while bsr.is_element_not_present_by_css("div.list-time"):
print "sleeping"
time.sleep(1)
bsr.find_by_css("span.iblock.icn-msg.list-icon.potr")[0].click()
page_num = get_page_num(bsr)
with open(out_path, "w") as out_f:
for pi in xrange(page_num):
print "Page %d/%d" % (pi+1, page_num)
date_lst = bsr.find_by_css("div.list-time")
date_msgs_lst = bsr.find_by_css("div.sms-item")
#HACK for scrolling the sms list down because of AJAX-style of showing sms
date_lst[-1].right_click()
msg_i = 0
for di in xrange(len(date_lst)):
date = date_lst[di].text.strip().split()[0]
msg_num_mat = re.findall(r"\(\s*(\d+).\s*\)", date_lst[di].text)
msg_num = int(msg_num_mat[0])
out_f.write("%s\t%d\n" % (date, msg_num))
for _ in range(msg_num):
name_obj = date_msgs_lst[msg_i].find_by_css("span.js-cnt.name")[0]
phone_obj = date_msgs_lst[msg_i].find_by_css("span.js-cnt.fc2")[0]
time_obj = date_msgs_lst[msg_i].find_by_css("div.fr.w6.js-cnt.bm-hack-w6")[0]
msg_obj = date_msgs_lst[msg_i].find_by_css("div.w4")[0]
type_obj = date_msgs_lst[msg_i].find_by_css("div.fl.w3.thide.fc5")[0]
out_f.write("%s\t%s\t%s\t%s\t%s\n" % (name_obj.html.encode("utf8"), \
phone_obj.html.strip("() ").encode("utf8"), \
time_obj.text.encode("utf8"), \
"0" if type_obj.visible else "1", \
msg_obj.text.encode("utf8")))
msg_i += 1
#next page
next_page_link = bsr.find_by_css("div.fr.pager")[0].find_by_tag("a")[2]
next_page_link.click()
def buyprd(prdurl):
browser = Browser('firefox')
browser.visit(prdurl)
time.sleep(.5)
print(browser.title)
browser.find_option_by_text(selectOption).first.click()
browser.find_by_name('commit').click()
if browser.is_text_present('item'):
print("Added to Cart")
else:
print("Error")
return
time.sleep(2)
print("checking out")
browser.visit(checkoutUrl)
time.sleep(.5)
print("Filling Out Billing Info")
browser.fill("order[billing_name]", namefield)
browser.fill("order[email]", emailfield)
browser.fill("order[tel]", phonefield)
print("Filling Out Address")
browser.fill("order[billing_address]", addressfield)
browser.fill("order[billing_zip]", zipfield)
browser.select("order[billing_state]", statefield)
print("Filling Out Credit Card Info")
browser.select("credit_card[type]", cctypefield)
browser.fill("credit_card[number]", ccnumfield)
browser.select("credit_card[month]", ccmonthfield)
browser.select("credit_card[year]", ccyearfield)
browser.fill("credit_card[verification_value]", cccvcfield)
browser.find_by_css('.terms').click()
time.sleep(.2)
print("Submitting Info")
browser.find_by_name('commit').click()
time.sleep(1)
browser.driver.save_screenshot('confirmation.png')
print("Exiting...")
time.sleep(2)
sys.exit(0)
class TestViews(unittest.TestCase):
def setUp(self):
""" Test setup """
self.browser = Browser("firefox")
# Set up the tables in the database
Base.metadata.create_all(engine)
# Create an example user
self.user = User(name="Alice", email="*****@*****.**",
password=generate_password_hash("test"))
session.add(self.user)
session.commit()
self.process = multiprocessing.Process(target=app.run,
kwargs={"port": 8080})
self.process.start()
time.sleep(1)
def tearDown(self):
""" Test teardown """
# Remove the tables and their data from the database
self.process.terminate()
session.close()
engine.dispose()
Base.metadata.drop_all(engine)
self.browser.quit()
def test_login_correct(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")
def test_login_incorrect(self):
self.browser.visit("http://127.0.0.1:8080/login")
self.browser.fill("email", "*****@*****.**")
self.browser.fill("password", "test")
button = self.browser.find_by_css("button[type=submit]")
button.click()
self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")
class SplinterTestCase(LiveServerTestCase):
username = '******'
email = '*****@*****.**'
password = '******'
is_anonymous = True
is_staff = False
is_logged_in = True
def setUp(self):
settings.DEBUG = True
super(SplinterTestCase, self).setUp()
self.user = None
self.base_url = URL(self.live_server_url)
self.browser = Browser(SPLINTER_WEBDRIVER)
if self.is_anonymous and not self.is_staff:
return
self.user = factories.UserFactory(
username=self.username,
email=self.email,
password=self.password,
is_staff=self.is_staff,
)
if self.is_logged_in:
self.goto(reverse('admin:index'))
self.browser.fill_form({
'username': self.username,
'password': self.password,
})
self.browser.find_by_css("input[type='submit']").first.click()
self.assertIn('Log out', self.browser.html)
def tearDown(self):
super(SplinterTestCase, self).tearDown()
self.browser.quit()
def goto(self, path):
url = self.base_url.path(path)
return self.browser.visit(url.as_string())
def run(self):
browser = Browser('chrome', executable_path="C:\Python27\Lib\site-packages\chromedriver_win32\chromedriver.exe", service_args=PROXIES)
# browser = Browser('phantomjs', service_args=PROXIES, user_agent="Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0")
with browser:
page = 1
browser.visit(self.url)
browser.fill("p", self.keyword)
browser.find_by_id("search-submit").click()
while True:
time.sleep(10)
logging.info("Page " + str(page))
for link in browser.find_by_css("div.res"):
if "applicationname" in link.find_by_css("a").first["href"].lower():
self.queue.put(link.find_by_css("a").first["href"])
page += 1
if browser.find_by_css("#pg-next"):
browser.find_by_css("#pg-next").click()
else:
break
self.queue.put(None)
def run():
browser = Browser('firefox', profile=config.FIREFOX_PROFILE_PATH)
browser.visit('http://www.tf2outpost.com/trades')
buttonList = browser.find_by_css(".trade_bump")
listSize = len(buttonList)
log.logMessage("Bumping " + str(listSize) + " items")
for i in range(listSize):
buttonList[i].click()
browser.quit()
