def main():
items = []
#Log in
webdriver.get(MAIN_PAGE_URL)
webdriver.find_element_by_id(
'ctl00_ContentPlaceHolder1_Username').send_keys(
credentials['username'])
webdriver.find_element_by_id(
'ctl00_ContentPlaceHolder1_Password').send_keys(
credentials['password'])
webdriver.find_element_by_name('ctl00$ContentPlaceHolder1$ctl04').click()
#Set items to show = 100
webdriver.find_element_by_xpath(
"//select[@name='ctl00$ContentPlaceHolder1$GridView1$ctl13$ctl11']/option[text()='100']"
).click()
#Getting number of pages
page_number = len(
webdriver.find_elements_by_xpath(
"//tr[@class='grid-pager']//table//tr/td[not(@class)]"))
page_href_script = "__doPostBack('ctl00$ContentPlaceHolder1$GridView1','Page$%s')"
#Extracting each page on the website
for i in range(page_number):
i += 1
if i != 1:
webdriver.execute_script(page_href_script % i)
# Wait for redirecting
time.sleep(10)
items += extract()
#Insert into database all extracted items (rigs)
insert_into_database(items)
def shot(cap, url, ads=None):
webdriver = webdriver.Remote(hubUrl,
desired_capabilities=cap,
proxy=proxy)
webdriver.set_window_size(1024, 800)
webdriver.get(url)
if ads:
sizeJs = "elm=document.getElementById(\"%s\");\
elm.style.display=\"block\";elm.style.width=\"%s px\";\
elm.style.height=\"%s px\";elm.style.overflow=\"hidden\";"
for ad in ads:
elm = webdriver.find_element_by_id(ad["id"])
if ad["size"] != (elm.size["width"], elm.size["height"]):
webdriver.execute_script(sizeJs % (ad["id"], ad["size"][0],
ad["size"][1]))
try:
ads = webdriver.execute_script("return pozice;")
except selenium.common.exceptions.WebDriverException:
ads = []
for ad in ads:
elm = webdriver.find_element_by_id(ad["id"])
ad["size"] = elm.size["width"], elm.size["height"]
ad["location"] = elm.location["x"], elm.location["y"]
for dId in debugIds:
webdriver.execute_script(removeDebug % dId)
_, filename = tempfile.mkstemp()
webdriver.save_screenshot(filename)
data = open(filename).read()
os.remove(filename)
return data, ads
def get_reviews(thisreview):
global last_len
print "Don't Stop"
for webdriver_obj in thisreview.find_elements_by_class_name("WMbnJf"):
Name = webdriver_obj.find_element_by_class_name("Y0uHMb")
Reviewer.append(Name.text)
try:
ReviewByuser = webdriver_obj.find_element_by_class_name("A503be")
TotalReviewsByUser.append(ReviewByuser.text)
except NoSuchElementException:
TotalReviewsByUser.append("")
star = webdriver_obj.find_element_by_class_name("fTKmHE99XE4__star")
ReviewStar = star.get_attribute("aria-label")
ReviewRating.append(ReviewStar)
Date = webdriver_obj.find_element_by_class_name("dehysf")
ReviewDate.append(Date.text)
Body = webdriver_obj.find_element_by_class_name('Jtu6Td')
try:
webdriver_obj.find_element_by_class_name('review-snippet').click()
s_32B = webdriver_obj.find_element_by_class_name(
'review-full-text')
ReviewDescription.append(s_32B.text)
except NoSuchElementException:
ReviewDescription.append(Body.text)
print("Yes..")
element = webdriver_obj.find_element_by_class_name('PuaHbe')
webdriver.execute_script("arguments[0].scrollIntoView();", element)
print("ah!..Go")
time.sleep(3)
reviews = webdriver.find_elements_by_class_name(
"gws-localreviews__general-reviews-block")
r_len = len(reviews)
if r_len > last_len:
last_len = r_len
get_reviews(reviews[r_len - 1])
def runFollowersFollow(self, webdriver, person, account, stopFlag):
sleep(2)
webdriver.get('https://www.%s.com/%s/followers/' %
(str.lower(account.accountType), person))
sleep(1)
follow_list = webdriver.find_elements_by_xpath(self.follow_list)
maxVal = len(follow_list)
followDoc = open("./bot_docs/masterList.txt", "a")
for index in range(1, 10):
try:
followButton = webdriver.find_element_by_xpath(
self.follow_list_item + str(index) + ']/div/div[3]/button')
accountName = webdriver.find_element_by_xpath(
self.follow_list_item + str(index) +
']/div/div[2]/div[1]/div/div/a')
webdriver.execute_script(
"return arguments[0].scrollIntoView();", followButton)
sleep(random.uniform(0, 1))
# if(followButton.text=='Follow' and accountName.text not in clickedList.split(',')[0]):
# followButton.click()
# followDoc.write(accountName.text + ', ' + accountName.get_attribute("href") + datetime.today().strftime('%Y-%m-%d') + '\n')
except:
print("no follow button: " + self.follow_list_item +
str(index) + ']/div/div[3]/button')
followDoc.close()
def url_smartclick(webdriver, url):
'''
Helps navigate around web elements that like to crash the scraper.
Attempts to click normally, and if the element is not in view it will force
the element into view, allowing it to be clicked.
--------------------------------------------------------------------------------------------
Input:
webdriver: Selenium webdriver object
url: Selenium "child" object, points to page you are currently on.
--------------------------------------------------------------------------------------------
Output:
NA: Function is for navigation, not returning values.
'''
try:
url.find_element_by_xpath(".//div[@class='indicator']").click()
print("url_smartclick," " 1")
except:
print("url_smartclick," " 2")
temp = webdriver.find_element_by_xpath('//div[@class="designers-popover"]')
temp = temp.find_element_by_xpath('.//div[@class="content"]')
temp_designers = temp.find_elements_by_xpath('.//div[@class="active-indicator"]')
webdriver.execute_script("arguments[0].scrollIntoView();", temp_desingers[-1])
#ActionChains(webdriver).move_to_element(temp_designers[-1])
url.find_element_by_xpath(".//div[@class='indicator']").click()
def scrape_page(webdriver, links, username):
'''This function will go to all links provided
and scrape each picture for the number of likes
and the caption. If the link is a video no information is recorded.
The function will only save the caption if the title is the
identified user
Parameters: the active webdriver, a set of picture links,
the username of the page your are scraping
Returns: a list of lists with the number of likes and caption
'''
picture_info = []
for link in links:
# Open new tab
webdriver.execute_script("window.open('');")
time.sleep(3)
# Switch to the new window
webdriver.switch_to.window(webdriver.window_handles[1])
webdriver.get(link)
time.sleep(5)
try:
likes_list = webdriver.find_elements_by_class_name('zV_Nj')
if len(likes_list) != 0: #If the length is 0, then it is a video
if len(likes_list) == 1: #No common friends liked the photo
num_likes = webdriver.find_elements_by_class_name('Nm9Fw')[0].text.split(' ')[0]
else:
num_likes = int(likes_list[1].text.split(' ')[0]) + 1
try:
title = webdriver.find_element_by_class_name('_6lAjh').text
if title == username:
caption_list = webdriver.find_elements_by_xpath("//div[@class='C4VMK']//span")
'''This code works but not active since I did not use the information
num_of_comments = len(caption_list)'''
caption = caption_list[0].text
else:
caption = None #if the user was not the title
except:
caption = None #photo does not have a caption or any comments
picture_info.append([num_likes, caption])
except:
pass
webdriver.close()
# Switch focus back to main tab
webdriver.switch_to.window(webdriver.window_handles[0])
time.sleep(5)
return picture_info
def scroll_timer(cls, webdriver):
'''Sets a timer for scrolling down the page'''
scrollheight = 1000
for timer in range(0, 50):
webdriver.execute_script("window.scrollTo(0, " +
str(scrollheight) + ")")
scrollheight += 1000
time.sleep(1)
def _perform_submit(webdriver=None):
submit_btn = webdriver.find_element_by_css_selector(
'input[type="submit"][name="_save"]')
webdriver.execute_script("arguments[0].click();", submit_btn)
WebDriverWait(webdriver,
10).until(EC.url_matches(f'{ORIGIN_URL}/patients/(?!new)'))
logger.info('Save patient successful')
def shot(webdriver, browser, testId, buildId):
proxy = ServerProxy(uploadUrl)
webdriver.set_window_size(1024, 800)
ret = proxy.remoteFile.read(libs.path.uploadPath(testId, browser,
"original",
"ads.pickle"))
if ret["status"] == 200:
originExists = True
else:
originExists = False
if not originExists:
ads1 = loads(ret["data"].data)
if ads1:
sizeJs = "elm=document.getElementById(\"%s\");\
elm.style.display=\"block\";\
elm.style.width=\"%s px\";elm.style.height=\"%s px\";\
elm.style.overflow=\"hidden\";"
for ad in ads1:
elm = webdriver.find_element_by_id(ad["id"])
if ad["size"] != (elm.size["width"], elm.size["height"]):
webdriver.execute_script(sizeJs % (ad["id"], ad["size"][0],
ad["size"][1]))
try:
ads2 = webdriver.execute_script("return pozice;")
except selenium.common.exceptions.WebDriverException:
ads2 = []
for ad in ads2:
elm = webdriver.find_element_by_id(ad["id"])
ad["size"] = elm.size["width"], elm.size["height"]
ad["location"] = elm.location["x"], elm.location["y"]
for dId in debugIds:
webdriver.execute_script(removeDebug % dId)
_, filename = tempfile.mkstemp()
webdriver.save_screenshot(filename)
data = open(filename).read()
os.remove(filename)
adsPickled = dumps(ads2)
if originExists:
proxy.remoteFile.write(libs.path.uploadPath(testId, browser, buildId,
"build.png"),
Binary(data), False)
proxy.remoteFile.write(libs.path.uploadPath(testId, browser, buildId,
"ads.pickle"),
Binary(adsPickled), False)
else:
proxy.remoteFile.write(libs.path.uploadPath(testId, browser,
"original",
"original.png"),
Binary(data), False)
proxy.remoteFile.write(libs.path.uploadPath(testId, browser,
"original",
"ads.pickle"),
Binary(adsPickled), False)
def publish_article_in_wechat(webdriver):
publish_btn_xpath = '//*[@id="js_send"]'
wait_element_visible(webdriver, publish_btn_xpath)
publish_btn = webdriver.find_element_by_xpath(publish_btn_xpath)
webdriver.execute_script('arguments[0].click()', publish_btn)
# publish_btn.click()
group_publish_btn_xpath = '//*[@id="send_btn_main"]/div/a/label'
wait_element_visible(webdriver, group_publish_btn_xpath)
group_publish_btn = webdriver.find_element_by_xpath(
'//*[@id="send_btn_main"]/div/a/label')
group_publish_btn.click()
def scroll_to_element(webdriver, element, scroll=-150):
"""
Brings the specified element into view
:input webdriver: The Selenium webdriver
:input element: The element to scroll to
"""
actions = ActionChains(webdriver)
actions.move_to_element(element).perform()
webdriver.execute_script(f"window.scrollBy(0, {scroll});")
time.sleep(1)
def transform_markdown_towechat(webdriver, content):
webdriver.get('http://blog.didispace.com/tools/online-markdown/')
content_input = webdriver.find_element_by_xpath('//*[@id="input"]')
content_input.clear()
content = content.replace('"',
'\\"').replace('\n', (' \\' + 'n ')).replace(
'\\"\\n\\"', '\\"\\\\n\\"')
# content = 'hello' + ' \\' + 'n' + '\\ ' + 'world'
dd = "arguments[0].value=\"%s\"" % content
print('content is ' + dd)
webdriver.execute_script(dd, content_input)
copy_btn = webdriver.find_element_by_xpath(
'//*[@id="output"]/div[2]/button')
copy_btn.click()
def probe_jqueryv2(webdriver):
try:
jquery = webdriver.execute_script("return jQuery.fn.jquery")
print "jquery: " + jquery
except:
#print 'probe_window FAILED'
pass
def get_controller_info(self, webdriver, controller_name):
print "GCI"
print controller_name
javascript = """var res = angular.element(document.body).injector().get('$controller');
var controller = res('""" + controller_name + """', {})
var rtnData = []
for (prop in controller) {
var mytype = typeof(controller[prop]);
if(mytype=="string" || mytype == "number" || mytype == "boolean"){
rtnData.push({'name': prop, 'value': controller[prop], 'type': mytype});
}
if(mytype=="object"){
rtnData.push({'name': prop, 'value': '', 'type': mytype});
}
}
return rtnData"""
try:
result = webdriver.execute_script(javascript)
except:
print "error@get_controller_info"
pass
return []
return result
def get_imgsrc_by_render(url, webdriver):
webdriver.set_page_load_timeout(60)
try:
webdriver.get(url)
except TimeoutException:
print("timeout")
webdriver.execute_script('window.stop()')
finally:
soup_html = BeautifulSoup(webdriver.page_source, 'lxml')
# print(webdriver.page_source)
for img_src in soup_html.find_all('img'):
if 'name' in img_src.attrs and img_src['name'] == 'TheImg':
print(img_src['src'])
return img_src['src']
with open(file_path, "w") as f:
f.write(webdriver.page_source)
def probe_jqueryv2(webdriver):
try:
jquery = webdriver.execute_script("return jQuery.fn.jquery")
print "jquery: "+jquery
except:
#print 'probe_window FAILED'
pass
def main_menu_items_collapsed(webdriver):
return webdriver.execute_script('''
rows = Sao.main_menu_screen.current_view.rows;
count = 0;
for (var i = 0; i < rows.length; i++) {
count += rows[i].rows.length;
}
return count == 0;''')
def do_testharness(self, webdriver, url, timeout):
format_map = {
"abs_url": url,
"url": strip_server(url),
"window_id": self.window_id,
"timeout_multiplier": self.timeout_multiplier,
"timeout": timeout * 1000
}
parent = webdriver.current_window_handle
handles = [item for item in webdriver.window_handles if item != parent]
for handle in handles:
try:
webdriver.switch_to_window(handle)
webdriver.close()
except exceptions.NoSuchWindowException:
pass
webdriver.switch_to_window(parent)
webdriver.execute_script(self.script % format_map)
try:
# Try this, it's in Level 1 but nothing supports it yet
win_s = webdriver.execute_script("return window['%s'];" %
self.window_id)
win_obj = json.loads(win_s)
test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"]
except Exception:
after = webdriver.window_handles
if len(after) == 2:
test_window = next(iter(set(after) - set([parent])))
elif after[0] == parent and len(after) > 2:
# Hope the first one here is the test window
test_window = after[1]
else:
raise Exception("unable to find test window")
assert test_window != parent
handler = CallbackHandler(webdriver, test_window, self.logger)
while True:
result = webdriver.execute_async_script(self.script_resume %
format_map)
done, rv = handler(result)
if done:
break
return rv
def launchWebpage(webdriver, htmlCode, htmlFileName):
global FILEHANDLE, fileOpened
htmlFileName += '.html'
path = os.path.abspath(htmlFileName)
url = 'file://' + path
print "Creating htmlfile with filename: " + url
fileOpened = True
with open(path, 'w') as FILEHANDLE:
FILEHANDLE.write(htmlCode)
webdriver.execute_script("window.open('','_blank');")
webdriver.switch_to.window(webdriver.window_handles[-1])
webdriver.get(url)
FILEHANDLE.close()
fileOpened = False
def isAngularApp(self, webdriver):
result = []
try:
result = webdriver.execute_script('return self.angular')
return len(result) > 0
except:
pass
return False
def do_testharness(self, webdriver, url, timeout):
format_map = {"abs_url": url,
"url": strip_server(url),
"window_id": self.window_id,
"timeout_multiplier": self.timeout_multiplier,
"timeout": timeout * 1000}
parent = webdriver.current_window_handle
handles = [item for item in webdriver.window_handles if item != parent]
for handle in handles:
try:
webdriver.switch_to_window(handle)
webdriver.close()
except exceptions.NoSuchWindowException:
pass
webdriver.switch_to_window(parent)
webdriver.execute_script(self.script % format_map)
try:
# Try this, it's in Level 1 but nothing supports it yet
win_s = webdriver.execute_script("return window['%s'];" % self.window_id)
win_obj = json.loads(win_s)
test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"]
except:
after = webdriver.window_handles
if len(after) == 2:
test_window = next(iter(set(after) - set([parent])))
elif after[0] == parent and len(after) > 2:
# Hope the first one here is the test window
test_window = after[1]
else:
raise Exception("unable to find test window")
assert test_window != parent
handler = CallbackHandler(webdriver, test_window, self.logger)
while True:
result = webdriver.execute_async_script(
self.script_resume % format_map)
done, rv = handler(result)
if done:
break
return rv
def login_complete(webdriver):
return webdriver.execute_script('''
try {
session_loaded = Boolean(
Sao.Session.current_session.session);
main_menu_loaded = Boolean(
Sao.main_menu_screen.current_view.rows.length);
return session_loaded && main_menu_loaded;
} catch(err) {
return false;
}''')
def get_screenshot(webdriver):
image_b64 = webdriver.execute_script(
"canvasRunner = document.getElementById('runner-canvas'); \
return canvasRunner.toDataURL().substring(22)")
screen = np.array(Image.open(BytesIO(base64.b64decode(image_b64))))
image = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
image = image[:300, :700]
image = cv2.inRange(image, 70, 90)
kernel = np.ones((5, 5), np.uint8)
image = cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel)
return cv2.resize(image, (config.DIMENSION, config.DIMENSION))
def capture():
seleniumlib = BuiltIn().get_library_instance('SeleniumLibrary')
webdriver = seleniumlib.driver
S = lambda X: webdriver.execute_script('return document.body.parentNode.scroll' + X)
webdriver.set_window_size(S('Width'), S('Height')) # May need manual adjustment
webdriver.find_element_by_tag_name('body').screenshot('image_name.png')
# capturing body tag gives full page screenshot
#body_tag = webdriver.find_element_by_tag_name('body')
#body_tag.screenshot('image_name.png')
def probe_window(webdriver):
print "getting global window object"
globalitems = []
noargfunctions = []
properrors = 0
try:
javascript = "jsproberesults=[];for (name in this) { try{jsproberesults.push( {'name':''+name, 'value': ''+this[name]})}catch(err){var anyerror='ignore'};}"
webdriver.execute_script(javascript)
javascript = "return jsproberesults"
jsresults = webdriver.execute_script(javascript)
for logline in jsresults:
if '[native code]' not in logline[
'value'] and 'jsproberesults' not in logline['name']:
globalitems.append(logline)
print str(len(globalitems)) + ' global items found'
for record in globalitems:
if record['value'].startswith('function ' + record['name'] +
'()') or record['value'].startswith(
'function ()'):
noargfunctions.append(record['name'])
print '\t' + record['name'] + ': ' + record['value']
print ""
print "found " + str(len(noargfunctions)) + " lone functions"
for record in noargfunctions:
print "\t" + record
print ""
print "Calling " + str(len(noargfunctions)) + " lone functions"
for record in noargfunctions:
print "\tCalling " + record
javascript = record + "();"
webdriver.execute_script(javascript)
except WebDriverException as e:
print "Selenium Exception: Message: " + str(e)
except:
print 'probe_window FAILED'
print "Unexpected error:", sys.exc_info()[0]
pass
def clickFollowButton(self, webdriver, val, stopFlag, file_object):
try:
follow_button = webdriver.find_element_by_xpath(
self.follow_list_item + str(val) + ']/button')
follow_id = webdriver.find_element_by_xpath(self.follow_list_item +
str(val) +
']/span/b/span/a')
if follow_id in self.followed_list:
return stopFlag
file_object.write(follow_id.text + '\n')
webdriver.execute_script("return arguments[0].scrollIntoView();",
follow_button)
webdriver.execute_script("window.scrollBy(0, -" +
str(random.randint(150, 200)) + ")")
sleep(random.uniform(.5, 6))
follow_button.click()
sleep(random.uniform(.5, 2))
follow_text = webdriver.find_element_by_xpath(
self.follow_list_item + str(val) + ']/button/span')
if follow_text.text == 'Follow':
print('Hit Max Follows Stopping')
stopFlag = False
return stopFlag
follow_text = webdriver.find_element_by_xpath(
self.follow_list_item + str(val) + ']/button/span')
webdriver.execute_script("window.scrollBy(0, -" +
str(random.randint(10, 50)) + ")")
name = webdriver.find_element_by_xpath(self.follow_list_item +
str(val) +
']/span/b/span/a')
except:
print("Follow Button Not Present: " + str(val))
return stopFlag
def probe_window(webdriver):
print "getting global window object"
globalitems=[]
noargfunctions=[]
properrors=0
try:
javascript="jsproberesults=[];for (name in this) { try{jsproberesults.push( {'name':''+name, 'value': ''+this[name]})}catch(err){var anyerror='ignore'};}"
webdriver.execute_script(javascript)
javascript="return jsproberesults"
jsresults = webdriver.execute_script(javascript)
for logline in jsresults:
if '[native code]' not in logline['value'] and 'jsproberesults' not in logline['name']:
globalitems.append(logline)
print str(len(globalitems))+' global items found'
for record in globalitems:
if record['value'].startswith('function '+record['name']+'()') or record['value'].startswith('function ()'):
noargfunctions.append(record['name'])
print '\t'+record['name']+': '+record['value']
print ""
print "found "+str(len(noargfunctions))+" lone functions"
for record in noargfunctions:
print "\t"+record
print ""
print "Calling "+str(len(noargfunctions))+" lone functions"
for record in noargfunctions:
print "\tCalling "+record
javascript = record+"();"
webdriver.execute_script(javascript)
except WebDriverException as e:
print "Selenium Exception: Message: "+str(e)
except:
print 'probe_window FAILED'
print "Unexpected error:", sys.exc_info()[0]
pass
def runFollowersFollow(self, webdriver, person, account, stopFlag):
sleep(2)
webdriver.get('https://www.%s.com/%s/followers/' %
(str.lower(account.accountType), person))
follow_list = webdriver.find_elements_by_xpath(self.follow_list)
index = 1
like_count = 0
file_object = open(
self.path + '/account-info/mixcloudFollowedList.txt', 'a')
while (like_count < int(account.numInteractions)):
for val in range(index, len(follow_list)):
if random.randint(0, 5) > 2 and stopFlag:
stopFlag = self.clickFollowButton(webdriver, val, stopFlag,
file_object)
if stopFlag:
like_count += 1
webdriver.execute_script("window.scrollBy(0, 200)")
if (like_count >= int(account.numInteractions)):
break
index = len(follow_list)
follow_list = webdriver.find_elements_by_xpath(self.follow_list)
file_object.close()
return stopFlag
def detect_routes_from_ui_router(self, webdriver):
rtnData = []
get_routes_javascript = """var rtnData = [];
var state = angular.element(document.body).injector().get('$state')
routes = state.get();
routes.forEach(function(value){rtnData.push({'url': value.url, 'controller': value.controller})});
return rtnData
"""
result = webdriver.execute_script(get_routes_javascript)
if len(result) > 0:
print "Routes Found:"
for url in result:
rtnData.append(url)
print "\t#%s - %s" % (url['url'], url['controller'])
print ''
return rtnData
def scrape_olr(neighborhood_code, webdriver):
username = os.getenv('JRX_USER')
password = os.getenv('JRX_PASSWORD')
if not username or not password:
raise Exception('The environment variables JRX_USER and JRX_PASSWORD must be set.')
username_input = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_txtUserName')
password_input = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_txtPassword')
username_input.send_keys(username)
time.sleep(1)
password_input.send_keys(password)
login_button = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_Loginexceed')
time.sleep(1)
login_button.click()
time.sleep(1)
customer_link = webdriver.find_element_by_xpath("//a[@data-label='View Customers']")
webdriver.execute_script("arguments[0].click();", customer_link)
time.sleep(2)
customer_grid = webdriver.find_element_by_xpath("//table[@id='ctl00_ctl00_MyCustContent_MyCustContent_gvParentAjax']")
jr_tblentry = customer_grid.find_element_by_xpath("//tr[@id='tr_788817']")
target_div = jr_tblentry.find_element_by_xpath("//div[@id='divDetail_788817']")
# click on expand button
expand_btn = customer_grid.find_element_by_xpath("//input[@id='imgExpColl_788817']")
webdriver.execute_script("arguments[0].click();", expand_btn)
time.sleep(1)
#nested_tables = target_div.find_elements_by_tag_name('table')
#print(dir(nested_tables))
#print('### %d nested tables found in target div.' % len(nested_tables))
#print('### found target table.')
search_link_condos = target_div.find_element_by_xpath("//a[@href='MyCustomerImageHndlr.aspx?Type=Runsearch&SearchID=956274&RunSearchType=AdvancedSale&cid=788817&From=View']")
webdriver.execute_script("arguments[0].click();", search_link_condos)
time.sleep(1)
top_div = webdriver.find_element_by_xpath("//div[@class='container_1130 search-ui']")
detail_areas = top_div.find_elements_by_xpath("//div[@class='apt_details_area clearfix']")
print('### %s listings found.' % len(detail_areas))
for da in detail_areas:
#apt_detail = da.find_element_by_xpath("//div[@class='apt_details_left']")
markup = da.get_attribute('innerHTML')
soup_parser = BeautifulSoup(markup, 'html.parser')
scrape_olr_condo_listings(soup_parser, markup)
#print(markup)
print('#################################################')
def getApplicationParts(self, webdriver):
rtnData = []
app_name = self.getAngularAppName(webdriver)
parts_javascript = """var parts=[];
angular.module('""" + app_name + """')['_invokeQueue'].forEach(function(value){
var items = value[2][1];
isarray = items instanceof Array;
if(isarray){
var newdic = items.slice(0, -1);
var source = items[items.length-1];
var record = {'parttype': value[1], 'name': value[2][0], 'components': newdic, 'sourcecode': source}
parts.push(record);
}
else{
var source = items;
var components = source.$inject
var record = {'parttype': value[1], 'name': value[2][0], 'sourcecode': source, 'components': components};
parts.push(record);
}
});
return parts"""
result = webdriver.execute_script(parts_javascript)
print "\tAngular Parts"
for part in result:
print "\t\t%s : %s ( %s )" % (part['parttype'], part['name'],
', '.join(part['components']))
for part in result:
urls_found = find_urls_from_source_code(part['sourcecode'])
if len(urls_found) > 0:
print "%s ( %s )" % (part['name'], ', '.join(
part['components']))
#print part['sourcecode']
for potential_url_string in urls_found:
print potential_url_string
print "-" * 100
for part in result:
for component in part['components']:
rtnData.append(component)
return rtnData
def debug(webdriver):
webdriver.switch_to.window(webdriver.window_handles[-1])
webdriver.get("http://www.isi.edu/~pedro/")
webdriver.execute_script("window.open('','_blank');")
webdriver.switch_to.window(webdriver.window_handles[-1])
webdriver.get("http://www-classes.usc.edu/engr/ee-s/477p/")
webdriver.execute_script("window.open('','_blank');")
webdriver.switch_to.window(webdriver.window_handles[-1])
webdriver.get("https://www.google.com/")
webdriver.execute_script("window.open('','_blank');")
webdriver.switch_to.window(webdriver.window_handles[-1])
webdriver.get("http://www.tv-porinternet.com/")
def _get_z_index(element, webdriver):
"""Tries to find the actual z-index of an element, otherwise returns 0."""
e = element
while e is not None:
try:
# selenium is usually wrong, don't bother with this
#z = element.value_of_css_property('z-index')
# get z-index with javascript
script = 'return window.document.defaultView.getComputedStyle(arguments[0], null).getPropertyValue("z-index")'
z = webdriver.execute_script(script, e)
if z != None and z != 'auto':
try:
return int(z)
except ValueError:
pass
# try the parent...
e = e.find_element_by_xpath(
'..') # throws exception when parent is the <html> tag
except:
break
return 0
def get_application_classitem(self, webdriver, classname_to_load):
rtnData = []
#ignore any classes starting with a dollar sign...
if classname_to_load.startswith('$'):
return []
if classname_to_load.endswith('Provider'):
print "oldCLASS " + classname_to_load
classname_to_load = classname_to_load[0:-8]
print "newCLASS " + classname_to_load
results = []
try:
javascript = """var res = angular.element(document.body).injector().get('""" + classname_to_load + """');
var rtnData = []
for (prop in res) {
var mytype = typeof(res[prop]);
if(mytype=="string" || mytype == "number" || mytype == "boolean"){
rtnData.push({'name': prop, 'value': res[prop], 'type': mytype});
}
if(mytype=="object"){
rtnData.push({'name': prop, 'value': '', 'type': mytype});
}
}
return rtnData"""
result = webdriver.execute_script(javascript)
if len(result) > 0:
for result_item in result:
rtnData.append(result_item)
except:
print "error@get_application_classitem"
pass
return rtnData
def sendDate(id, txt):
if (len(txt) > 0):
splitDay = txt.split("/")
gregorian_date = jdatetime.date(int(splitDay[0]), int(splitDay[1]),
int(splitDay[2])).togregorian()
day = webdriver.find_element_by_xpath(
"/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div["
+ str(id) + "]/span[1]/input[1]")
month = webdriver.find_element_by_xpath(
"/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div["
+ str(id) + "]/span[1]/input[2]")
year = webdriver.find_element_by_xpath(
"/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div["
+ str(id) + "]/span[1]/input[3]")
webdriver.execute_script(
"arguments[0].setAttribute('value', '" + str(gregorian_date.day) +
"')", day)
webdriver.execute_script(
"arguments[0].setAttribute('value', '" +
str(gregorian_date.month - 1) + "')", month)
webdriver.execute_script(
"arguments[0].setAttribute('value', '" + str(gregorian_date.year) +
"')", year)
def probe_DARLA_CONFIG(webdriver):
try:
darla = webdriver.execute_script("return DARLA_CONFIG.version")
print "DARLA_CONFIG: "+darla
except:
pass
def probe_backbone(webdriver):
try:
backbone = webdriver.execute_script("return Backbone.VERSION")
print "Backbone: "+backbone
except:
pass
f = open("/home/dhl/shipment_numbers.csv")
wait = WebDriverWait(webdriver, 60 * 60)
for line in f:
#Searching for shipment code
webdriver.find_element_by_id('shipmentCode').send_keys(line)
webdriver.find_element_by_id('timeIntervall').send_keys('12')
webdriver.find_element_by_name('search_ta').click()
#Downloading pdf
try:
actionSelection = wait.until(EC.element_to_be_clickable((By.ID,'pageActionSelect')))
finally:
pass
actionSelection.send_keys('download')
webdriver.execute_script('javascript:openLayerSelection(\'pieceList\',\'false\')')
#webdriver.find_element_by_id('signchk').click()
try:
element = wait.until(EC.element_to_be_clickable((By.ID,'signchk')))
finally:
pass
element.click()
webdriver.find_element_by_xpath("//input[@value='Download']").click()
webdriver.implicitly_wait(600)
webdriver.find_element_by_name('cancel_ta').click()
print "downloaded shipment pdf with shipment code :" + line
f.close()
def probe_drupal(webdriver):
try:
data = webdriver.execute_script("return Drupal")
print "Drupal CMS detected"
except:
pass
def probe_modernizr(webdriver):
try:
modenizer = webdriver.execute_script("return Modernizr._version")
print "Modernizr: "+modenizer
except:
pass
def probe_microsoft_dynamics_crm(webdriver):
try:
serverurl = webdriver.execute_script("return SERVER_URL")
userlangcode = webdriver.execute_script("return USER_LANGUAGE_CODE")
orglangcode = webdriver.execute_script("return ORG_LANGUAGE_CODE")
orgname = webdriver.execute_script("return ORG_UNIQUE_NAME")
appversion = webdriver.execute_script("return APPLICATION_FULL_VERSION")
# if we got this far without an error then we must be crm
print "Microsoft Dynamics CRM found:"
print "\tSERVER_URL: "+serverurl
print "\tUSER_LANGUAGE_CODE: "+str(userlangcode)
print "\tORG_LANGUAGE_CODE: "+str(orglangcode)
print "\tORG_UNIQUE_NAME: "+orgname
print "\tAPPLICATION_FULL_VERSION: "+appversion
#bonus
print "\tIS_PATHBASEDURLS: "+str(webdriver.execute_script("return IS_PATHBASEDURLS"))
print "\tIS_OUTLOOK_CLIENT: "+str(webdriver.execute_script("return IS_OUTLOOK_CLIENT"))
print "\tIS_ONLINE: "+str(webdriver.execute_script("return IS_ONLINE"))
print "\tIS_LIVE: "+str(webdriver.execute_script("return IS_LIVE"))
print "\tIS_ONPREMISE: "+str(webdriver.execute_script("return IS_ONPREMISE"))
print "\tCURRENT_WEB_THEME: "+str(webdriver.execute_script("return CURRENT_WEB_THEME"))
print ""
except:
pass
