def main(): items = [] #Log in webdriver.get(MAIN_PAGE_URL) webdriver.find_element_by_id( 'ctl00_ContentPlaceHolder1_Username').send_keys( credentials['username']) webdriver.find_element_by_id( 'ctl00_ContentPlaceHolder1_Password').send_keys( credentials['password']) webdriver.find_element_by_name('ctl00$ContentPlaceHolder1$ctl04').click() #Set items to show = 100 webdriver.find_element_by_xpath( "//select[@name='ctl00$ContentPlaceHolder1$GridView1$ctl13$ctl11']/option[text()='100']" ).click() #Getting number of pages page_number = len( webdriver.find_elements_by_xpath( "//tr[@class='grid-pager']//table//tr/td[not(@class)]")) page_href_script = "__doPostBack('ctl00$ContentPlaceHolder1$GridView1','Page$%s')" #Extracting each page on the website for i in range(page_number): i += 1 if i != 1: webdriver.execute_script(page_href_script % i) # Wait for redirecting time.sleep(10) items += extract() #Insert into database all extracted items (rigs) insert_into_database(items)
def shot(cap, url, ads=None): webdriver = webdriver.Remote(hubUrl, desired_capabilities=cap, proxy=proxy) webdriver.set_window_size(1024, 800) webdriver.get(url) if ads: sizeJs = "elm=document.getElementById(\"%s\");\ elm.style.display=\"block\";elm.style.width=\"%s px\";\ elm.style.height=\"%s px\";elm.style.overflow=\"hidden\";" for ad in ads: elm = webdriver.find_element_by_id(ad["id"]) if ad["size"] != (elm.size["width"], elm.size["height"]): webdriver.execute_script(sizeJs % (ad["id"], ad["size"][0], ad["size"][1])) try: ads = webdriver.execute_script("return pozice;") except selenium.common.exceptions.WebDriverException: ads = [] for ad in ads: elm = webdriver.find_element_by_id(ad["id"]) ad["size"] = elm.size["width"], elm.size["height"] ad["location"] = elm.location["x"], elm.location["y"] for dId in debugIds: webdriver.execute_script(removeDebug % dId) _, filename = tempfile.mkstemp() webdriver.save_screenshot(filename) data = open(filename).read() os.remove(filename) return data, ads
def get_reviews(thisreview): global last_len print "Don't Stop" for webdriver_obj in thisreview.find_elements_by_class_name("WMbnJf"): Name = webdriver_obj.find_element_by_class_name("Y0uHMb") Reviewer.append(Name.text) try: ReviewByuser = webdriver_obj.find_element_by_class_name("A503be") TotalReviewsByUser.append(ReviewByuser.text) except NoSuchElementException: TotalReviewsByUser.append("") star = webdriver_obj.find_element_by_class_name("fTKmHE99XE4__star") ReviewStar = star.get_attribute("aria-label") ReviewRating.append(ReviewStar) Date = webdriver_obj.find_element_by_class_name("dehysf") ReviewDate.append(Date.text) Body = webdriver_obj.find_element_by_class_name('Jtu6Td') try: webdriver_obj.find_element_by_class_name('review-snippet').click() s_32B = webdriver_obj.find_element_by_class_name( 'review-full-text') ReviewDescription.append(s_32B.text) except NoSuchElementException: ReviewDescription.append(Body.text) print("Yes..") element = webdriver_obj.find_element_by_class_name('PuaHbe') webdriver.execute_script("arguments[0].scrollIntoView();", element) print("ah!..Go") time.sleep(3) reviews = webdriver.find_elements_by_class_name( "gws-localreviews__general-reviews-block") r_len = len(reviews) if r_len > last_len: last_len = r_len get_reviews(reviews[r_len - 1])
def runFollowersFollow(self, webdriver, person, account, stopFlag): sleep(2) webdriver.get('https://www.%s.com/%s/followers/' % (str.lower(account.accountType), person)) sleep(1) follow_list = webdriver.find_elements_by_xpath(self.follow_list) maxVal = len(follow_list) followDoc = open("./bot_docs/masterList.txt", "a") for index in range(1, 10): try: followButton = webdriver.find_element_by_xpath( self.follow_list_item + str(index) + ']/div/div[3]/button') accountName = webdriver.find_element_by_xpath( self.follow_list_item + str(index) + ']/div/div[2]/div[1]/div/div/a') webdriver.execute_script( "return arguments[0].scrollIntoView();", followButton) sleep(random.uniform(0, 1)) # if(followButton.text=='Follow' and accountName.text not in clickedList.split(',')[0]): # followButton.click() # followDoc.write(accountName.text + ', ' + accountName.get_attribute("href") + datetime.today().strftime('%Y-%m-%d') + '\n') except: print("no follow button: " + self.follow_list_item + str(index) + ']/div/div[3]/button') followDoc.close()
def url_smartclick(webdriver, url): ''' Helps navigate around web elements that like to crash the scraper. Attempts to click normally, and if the element is not in view it will force the element into view, allowing it to be clicked. -------------------------------------------------------------------------------------------- Input: webdriver: Selenium webdriver object url: Selenium "child" object, points to page you are currently on. -------------------------------------------------------------------------------------------- Output: NA: Function is for navigation, not returning values. ''' try: url.find_element_by_xpath(".//div[@class='indicator']").click() print("url_smartclick," " 1") except: print("url_smartclick," " 2") temp = webdriver.find_element_by_xpath('//div[@class="designers-popover"]') temp = temp.find_element_by_xpath('.//div[@class="content"]') temp_designers = temp.find_elements_by_xpath('.//div[@class="active-indicator"]') webdriver.execute_script("arguments[0].scrollIntoView();", temp_desingers[-1]) #ActionChains(webdriver).move_to_element(temp_designers[-1]) url.find_element_by_xpath(".//div[@class='indicator']").click()
def scrape_page(webdriver, links, username): '''This function will go to all links provided and scrape each picture for the number of likes and the caption. If the link is a video no information is recorded. The function will only save the caption if the title is the identified user Parameters: the active webdriver, a set of picture links, the username of the page your are scraping Returns: a list of lists with the number of likes and caption ''' picture_info = [] for link in links: # Open new tab webdriver.execute_script("window.open('');") time.sleep(3) # Switch to the new window webdriver.switch_to.window(webdriver.window_handles[1]) webdriver.get(link) time.sleep(5) try: likes_list = webdriver.find_elements_by_class_name('zV_Nj') if len(likes_list) != 0: #If the length is 0, then it is a video if len(likes_list) == 1: #No common friends liked the photo num_likes = webdriver.find_elements_by_class_name('Nm9Fw')[0].text.split(' ')[0] else: num_likes = int(likes_list[1].text.split(' ')[0]) + 1 try: title = webdriver.find_element_by_class_name('_6lAjh').text if title == username: caption_list = webdriver.find_elements_by_xpath("//div[@class='C4VMK']//span") '''This code works but not active since I did not use the information num_of_comments = len(caption_list)''' caption = caption_list[0].text else: caption = None #if the user was not the title except: caption = None #photo does not have a caption or any comments picture_info.append([num_likes, caption]) except: pass webdriver.close() # Switch focus back to main tab webdriver.switch_to.window(webdriver.window_handles[0]) time.sleep(5) return picture_info
def scroll_timer(cls, webdriver): '''Sets a timer for scrolling down the page''' scrollheight = 1000 for timer in range(0, 50): webdriver.execute_script("window.scrollTo(0, " + str(scrollheight) + ")") scrollheight += 1000 time.sleep(1)
def _perform_submit(webdriver=None): submit_btn = webdriver.find_element_by_css_selector( 'input[type="submit"][name="_save"]') webdriver.execute_script("arguments[0].click();", submit_btn) WebDriverWait(webdriver, 10).until(EC.url_matches(f'{ORIGIN_URL}/patients/(?!new)')) logger.info('Save patient successful')
def shot(webdriver, browser, testId, buildId): proxy = ServerProxy(uploadUrl) webdriver.set_window_size(1024, 800) ret = proxy.remoteFile.read(libs.path.uploadPath(testId, browser, "original", "ads.pickle")) if ret["status"] == 200: originExists = True else: originExists = False if not originExists: ads1 = loads(ret["data"].data) if ads1: sizeJs = "elm=document.getElementById(\"%s\");\ elm.style.display=\"block\";\ elm.style.width=\"%s px\";elm.style.height=\"%s px\";\ elm.style.overflow=\"hidden\";" for ad in ads1: elm = webdriver.find_element_by_id(ad["id"]) if ad["size"] != (elm.size["width"], elm.size["height"]): webdriver.execute_script(sizeJs % (ad["id"], ad["size"][0], ad["size"][1])) try: ads2 = webdriver.execute_script("return pozice;") except selenium.common.exceptions.WebDriverException: ads2 = [] for ad in ads2: elm = webdriver.find_element_by_id(ad["id"]) ad["size"] = elm.size["width"], elm.size["height"] ad["location"] = elm.location["x"], elm.location["y"] for dId in debugIds: webdriver.execute_script(removeDebug % dId) _, filename = tempfile.mkstemp() webdriver.save_screenshot(filename) data = open(filename).read() os.remove(filename) adsPickled = dumps(ads2) if originExists: proxy.remoteFile.write(libs.path.uploadPath(testId, browser, buildId, "build.png"), Binary(data), False) proxy.remoteFile.write(libs.path.uploadPath(testId, browser, buildId, "ads.pickle"), Binary(adsPickled), False) else: proxy.remoteFile.write(libs.path.uploadPath(testId, browser, "original", "original.png"), Binary(data), False) proxy.remoteFile.write(libs.path.uploadPath(testId, browser, "original", "ads.pickle"), Binary(adsPickled), False)
def publish_article_in_wechat(webdriver): publish_btn_xpath = '//*[@id="js_send"]' wait_element_visible(webdriver, publish_btn_xpath) publish_btn = webdriver.find_element_by_xpath(publish_btn_xpath) webdriver.execute_script('arguments[0].click()', publish_btn) # publish_btn.click() group_publish_btn_xpath = '//*[@id="send_btn_main"]/div/a/label' wait_element_visible(webdriver, group_publish_btn_xpath) group_publish_btn = webdriver.find_element_by_xpath( '//*[@id="send_btn_main"]/div/a/label') group_publish_btn.click()
def scroll_to_element(webdriver, element, scroll=-150): """ Brings the specified element into view :input webdriver: The Selenium webdriver :input element: The element to scroll to """ actions = ActionChains(webdriver) actions.move_to_element(element).perform() webdriver.execute_script(f"window.scrollBy(0, {scroll});") time.sleep(1)
def transform_markdown_towechat(webdriver, content): webdriver.get('http://blog.didispace.com/tools/online-markdown/') content_input = webdriver.find_element_by_xpath('//*[@id="input"]') content_input.clear() content = content.replace('"', '\\"').replace('\n', (' \\' + 'n ')).replace( '\\"\\n\\"', '\\"\\\\n\\"') # content = 'hello' + ' \\' + 'n' + '\\ ' + 'world' dd = "arguments[0].value=\"%s\"" % content print('content is ' + dd) webdriver.execute_script(dd, content_input) copy_btn = webdriver.find_element_by_xpath( '//*[@id="output"]/div[2]/button') copy_btn.click()
def probe_jqueryv2(webdriver): try: jquery = webdriver.execute_script("return jQuery.fn.jquery") print "jquery: " + jquery except: #print 'probe_window FAILED' pass
def get_controller_info(self, webdriver, controller_name): print "GCI" print controller_name javascript = """var res = angular.element(document.body).injector().get('$controller'); var controller = res('""" + controller_name + """', {}) var rtnData = [] for (prop in controller) { var mytype = typeof(controller[prop]); if(mytype=="string" || mytype == "number" || mytype == "boolean"){ rtnData.push({'name': prop, 'value': controller[prop], 'type': mytype}); } if(mytype=="object"){ rtnData.push({'name': prop, 'value': '', 'type': mytype}); } } return rtnData""" try: result = webdriver.execute_script(javascript) except: print "error@get_controller_info" pass return [] return result
def get_imgsrc_by_render(url, webdriver): webdriver.set_page_load_timeout(60) try: webdriver.get(url) except TimeoutException: print("timeout") webdriver.execute_script('window.stop()') finally: soup_html = BeautifulSoup(webdriver.page_source, 'lxml') # print(webdriver.page_source) for img_src in soup_html.find_all('img'): if 'name' in img_src.attrs and img_src['name'] == 'TheImg': print(img_src['src']) return img_src['src'] with open(file_path, "w") as f: f.write(webdriver.page_source)
def probe_jqueryv2(webdriver): try: jquery = webdriver.execute_script("return jQuery.fn.jquery") print "jquery: "+jquery except: #print 'probe_window FAILED' pass
def main_menu_items_collapsed(webdriver): return webdriver.execute_script(''' rows = Sao.main_menu_screen.current_view.rows; count = 0; for (var i = 0; i < rows.length; i++) { count += rows[i].rows.length; } return count == 0;''')
def do_testharness(self, webdriver, url, timeout): format_map = { "abs_url": url, "url": strip_server(url), "window_id": self.window_id, "timeout_multiplier": self.timeout_multiplier, "timeout": timeout * 1000 } parent = webdriver.current_window_handle handles = [item for item in webdriver.window_handles if item != parent] for handle in handles: try: webdriver.switch_to_window(handle) webdriver.close() except exceptions.NoSuchWindowException: pass webdriver.switch_to_window(parent) webdriver.execute_script(self.script % format_map) try: # Try this, it's in Level 1 but nothing supports it yet win_s = webdriver.execute_script("return window['%s'];" % self.window_id) win_obj = json.loads(win_s) test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"] except Exception: after = webdriver.window_handles if len(after) == 2: test_window = next(iter(set(after) - set([parent]))) elif after[0] == parent and len(after) > 2: # Hope the first one here is the test window test_window = after[1] else: raise Exception("unable to find test window") assert test_window != parent handler = CallbackHandler(webdriver, test_window, self.logger) while True: result = webdriver.execute_async_script(self.script_resume % format_map) done, rv = handler(result) if done: break return rv
def launchWebpage(webdriver, htmlCode, htmlFileName): global FILEHANDLE, fileOpened htmlFileName += '.html' path = os.path.abspath(htmlFileName) url = 'file://' + path print "Creating htmlfile with filename: " + url fileOpened = True with open(path, 'w') as FILEHANDLE: FILEHANDLE.write(htmlCode) webdriver.execute_script("window.open('','_blank');") webdriver.switch_to.window(webdriver.window_handles[-1]) webdriver.get(url) FILEHANDLE.close() fileOpened = False
def isAngularApp(self, webdriver): result = [] try: result = webdriver.execute_script('return self.angular') return len(result) > 0 except: pass return False
def do_testharness(self, webdriver, url, timeout): format_map = {"abs_url": url, "url": strip_server(url), "window_id": self.window_id, "timeout_multiplier": self.timeout_multiplier, "timeout": timeout * 1000} parent = webdriver.current_window_handle handles = [item for item in webdriver.window_handles if item != parent] for handle in handles: try: webdriver.switch_to_window(handle) webdriver.close() except exceptions.NoSuchWindowException: pass webdriver.switch_to_window(parent) webdriver.execute_script(self.script % format_map) try: # Try this, it's in Level 1 but nothing supports it yet win_s = webdriver.execute_script("return window['%s'];" % self.window_id) win_obj = json.loads(win_s) test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"] except: after = webdriver.window_handles if len(after) == 2: test_window = next(iter(set(after) - set([parent]))) elif after[0] == parent and len(after) > 2: # Hope the first one here is the test window test_window = after[1] else: raise Exception("unable to find test window") assert test_window != parent handler = CallbackHandler(webdriver, test_window, self.logger) while True: result = webdriver.execute_async_script( self.script_resume % format_map) done, rv = handler(result) if done: break return rv
def login_complete(webdriver): return webdriver.execute_script(''' try { session_loaded = Boolean( Sao.Session.current_session.session); main_menu_loaded = Boolean( Sao.main_menu_screen.current_view.rows.length); return session_loaded && main_menu_loaded; } catch(err) { return false; }''')
def get_screenshot(webdriver): image_b64 = webdriver.execute_script( "canvasRunner = document.getElementById('runner-canvas'); \ return canvasRunner.toDataURL().substring(22)") screen = np.array(Image.open(BytesIO(base64.b64decode(image_b64)))) image = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) image = image[:300, :700] image = cv2.inRange(image, 70, 90) kernel = np.ones((5, 5), np.uint8) image = cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel) return cv2.resize(image, (config.DIMENSION, config.DIMENSION))
def capture(): seleniumlib = BuiltIn().get_library_instance('SeleniumLibrary') webdriver = seleniumlib.driver S = lambda X: webdriver.execute_script('return document.body.parentNode.scroll' + X) webdriver.set_window_size(S('Width'), S('Height')) # May need manual adjustment webdriver.find_element_by_tag_name('body').screenshot('image_name.png') # capturing body tag gives full page screenshot #body_tag = webdriver.find_element_by_tag_name('body') #body_tag.screenshot('image_name.png')
def probe_window(webdriver): print "getting global window object" globalitems = [] noargfunctions = [] properrors = 0 try: javascript = "jsproberesults=[];for (name in this) { try{jsproberesults.push( {'name':''+name, 'value': ''+this[name]})}catch(err){var anyerror='ignore'};}" webdriver.execute_script(javascript) javascript = "return jsproberesults" jsresults = webdriver.execute_script(javascript) for logline in jsresults: if '[native code]' not in logline[ 'value'] and 'jsproberesults' not in logline['name']: globalitems.append(logline) print str(len(globalitems)) + ' global items found' for record in globalitems: if record['value'].startswith('function ' + record['name'] + '()') or record['value'].startswith( 'function ()'): noargfunctions.append(record['name']) print '\t' + record['name'] + ': ' + record['value'] print "" print "found " + str(len(noargfunctions)) + " lone functions" for record in noargfunctions: print "\t" + record print "" print "Calling " + str(len(noargfunctions)) + " lone functions" for record in noargfunctions: print "\tCalling " + record javascript = record + "();" webdriver.execute_script(javascript) except WebDriverException as e: print "Selenium Exception: Message: " + str(e) except: print 'probe_window FAILED' print "Unexpected error:", sys.exc_info()[0] pass
def clickFollowButton(self, webdriver, val, stopFlag, file_object): try: follow_button = webdriver.find_element_by_xpath( self.follow_list_item + str(val) + ']/button') follow_id = webdriver.find_element_by_xpath(self.follow_list_item + str(val) + ']/span/b/span/a') if follow_id in self.followed_list: return stopFlag file_object.write(follow_id.text + '\n') webdriver.execute_script("return arguments[0].scrollIntoView();", follow_button) webdriver.execute_script("window.scrollBy(0, -" + str(random.randint(150, 200)) + ")") sleep(random.uniform(.5, 6)) follow_button.click() sleep(random.uniform(.5, 2)) follow_text = webdriver.find_element_by_xpath( self.follow_list_item + str(val) + ']/button/span') if follow_text.text == 'Follow': print('Hit Max Follows Stopping') stopFlag = False return stopFlag follow_text = webdriver.find_element_by_xpath( self.follow_list_item + str(val) + ']/button/span') webdriver.execute_script("window.scrollBy(0, -" + str(random.randint(10, 50)) + ")") name = webdriver.find_element_by_xpath(self.follow_list_item + str(val) + ']/span/b/span/a') except: print("Follow Button Not Present: " + str(val)) return stopFlag
def probe_window(webdriver): print "getting global window object" globalitems=[] noargfunctions=[] properrors=0 try: javascript="jsproberesults=[];for (name in this) { try{jsproberesults.push( {'name':''+name, 'value': ''+this[name]})}catch(err){var anyerror='ignore'};}" webdriver.execute_script(javascript) javascript="return jsproberesults" jsresults = webdriver.execute_script(javascript) for logline in jsresults: if '[native code]' not in logline['value'] and 'jsproberesults' not in logline['name']: globalitems.append(logline) print str(len(globalitems))+' global items found' for record in globalitems: if record['value'].startswith('function '+record['name']+'()') or record['value'].startswith('function ()'): noargfunctions.append(record['name']) print '\t'+record['name']+': '+record['value'] print "" print "found "+str(len(noargfunctions))+" lone functions" for record in noargfunctions: print "\t"+record print "" print "Calling "+str(len(noargfunctions))+" lone functions" for record in noargfunctions: print "\tCalling "+record javascript = record+"();" webdriver.execute_script(javascript) except WebDriverException as e: print "Selenium Exception: Message: "+str(e) except: print 'probe_window FAILED' print "Unexpected error:", sys.exc_info()[0] pass
def runFollowersFollow(self, webdriver, person, account, stopFlag): sleep(2) webdriver.get('https://www.%s.com/%s/followers/' % (str.lower(account.accountType), person)) follow_list = webdriver.find_elements_by_xpath(self.follow_list) index = 1 like_count = 0 file_object = open( self.path + '/account-info/mixcloudFollowedList.txt', 'a') while (like_count < int(account.numInteractions)): for val in range(index, len(follow_list)): if random.randint(0, 5) > 2 and stopFlag: stopFlag = self.clickFollowButton(webdriver, val, stopFlag, file_object) if stopFlag: like_count += 1 webdriver.execute_script("window.scrollBy(0, 200)") if (like_count >= int(account.numInteractions)): break index = len(follow_list) follow_list = webdriver.find_elements_by_xpath(self.follow_list) file_object.close() return stopFlag
def detect_routes_from_ui_router(self, webdriver): rtnData = [] get_routes_javascript = """var rtnData = []; var state = angular.element(document.body).injector().get('$state') routes = state.get(); routes.forEach(function(value){rtnData.push({'url': value.url, 'controller': value.controller})}); return rtnData """ result = webdriver.execute_script(get_routes_javascript) if len(result) > 0: print "Routes Found:" for url in result: rtnData.append(url) print "\t#%s - %s" % (url['url'], url['controller']) print '' return rtnData
def scrape_olr(neighborhood_code, webdriver): username = os.getenv('JRX_USER') password = os.getenv('JRX_PASSWORD') if not username or not password: raise Exception('The environment variables JRX_USER and JRX_PASSWORD must be set.') username_input = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_txtUserName') password_input = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_txtPassword') username_input.send_keys(username) time.sleep(1) password_input.send_keys(password) login_button = webdriver.find_element_by_id('ctl00_ContentPlaceHolder1_Loginexceed') time.sleep(1) login_button.click() time.sleep(1) customer_link = webdriver.find_element_by_xpath("//a[@data-label='View Customers']") webdriver.execute_script("arguments[0].click();", customer_link) time.sleep(2) customer_grid = webdriver.find_element_by_xpath("//table[@id='ctl00_ctl00_MyCustContent_MyCustContent_gvParentAjax']") jr_tblentry = customer_grid.find_element_by_xpath("//tr[@id='tr_788817']") target_div = jr_tblentry.find_element_by_xpath("//div[@id='divDetail_788817']") # click on expand button expand_btn = customer_grid.find_element_by_xpath("//input[@id='imgExpColl_788817']") webdriver.execute_script("arguments[0].click();", expand_btn) time.sleep(1) #nested_tables = target_div.find_elements_by_tag_name('table') #print(dir(nested_tables)) #print('### %d nested tables found in target div.' % len(nested_tables)) #print('### found target table.') search_link_condos = target_div.find_element_by_xpath("//a[@href='MyCustomerImageHndlr.aspx?Type=Runsearch&SearchID=956274&RunSearchType=AdvancedSale&cid=788817&From=View']") webdriver.execute_script("arguments[0].click();", search_link_condos) time.sleep(1) top_div = webdriver.find_element_by_xpath("//div[@class='container_1130 search-ui']") detail_areas = top_div.find_elements_by_xpath("//div[@class='apt_details_area clearfix']") print('### %s listings found.' % len(detail_areas)) for da in detail_areas: #apt_detail = da.find_element_by_xpath("//div[@class='apt_details_left']") markup = da.get_attribute('innerHTML') soup_parser = BeautifulSoup(markup, 'html.parser') scrape_olr_condo_listings(soup_parser, markup) #print(markup) print('#################################################')
def getApplicationParts(self, webdriver): rtnData = [] app_name = self.getAngularAppName(webdriver) parts_javascript = """var parts=[]; angular.module('""" + app_name + """')['_invokeQueue'].forEach(function(value){ var items = value[2][1]; isarray = items instanceof Array; if(isarray){ var newdic = items.slice(0, -1); var source = items[items.length-1]; var record = {'parttype': value[1], 'name': value[2][0], 'components': newdic, 'sourcecode': source} parts.push(record); } else{ var source = items; var components = source.$inject var record = {'parttype': value[1], 'name': value[2][0], 'sourcecode': source, 'components': components}; parts.push(record); } }); return parts""" result = webdriver.execute_script(parts_javascript) print "\tAngular Parts" for part in result: print "\t\t%s : %s ( %s )" % (part['parttype'], part['name'], ', '.join(part['components'])) for part in result: urls_found = find_urls_from_source_code(part['sourcecode']) if len(urls_found) > 0: print "%s ( %s )" % (part['name'], ', '.join( part['components'])) #print part['sourcecode'] for potential_url_string in urls_found: print potential_url_string print "-" * 100 for part in result: for component in part['components']: rtnData.append(component) return rtnData
def debug(webdriver): webdriver.switch_to.window(webdriver.window_handles[-1]) webdriver.get("http://www.isi.edu/~pedro/") webdriver.execute_script("window.open('','_blank');") webdriver.switch_to.window(webdriver.window_handles[-1]) webdriver.get("http://www-classes.usc.edu/engr/ee-s/477p/") webdriver.execute_script("window.open('','_blank');") webdriver.switch_to.window(webdriver.window_handles[-1]) webdriver.get("https://www.google.com/") webdriver.execute_script("window.open('','_blank');") webdriver.switch_to.window(webdriver.window_handles[-1]) webdriver.get("http://www.tv-porinternet.com/")
def _get_z_index(element, webdriver): """Tries to find the actual z-index of an element, otherwise returns 0.""" e = element while e is not None: try: # selenium is usually wrong, don't bother with this #z = element.value_of_css_property('z-index') # get z-index with javascript script = 'return window.document.defaultView.getComputedStyle(arguments[0], null).getPropertyValue("z-index")' z = webdriver.execute_script(script, e) if z != None and z != 'auto': try: return int(z) except ValueError: pass # try the parent... e = e.find_element_by_xpath( '..') # throws exception when parent is the <html> tag except: break return 0
def get_application_classitem(self, webdriver, classname_to_load): rtnData = [] #ignore any classes starting with a dollar sign... if classname_to_load.startswith('$'): return [] if classname_to_load.endswith('Provider'): print "oldCLASS " + classname_to_load classname_to_load = classname_to_load[0:-8] print "newCLASS " + classname_to_load results = [] try: javascript = """var res = angular.element(document.body).injector().get('""" + classname_to_load + """'); var rtnData = [] for (prop in res) { var mytype = typeof(res[prop]); if(mytype=="string" || mytype == "number" || mytype == "boolean"){ rtnData.push({'name': prop, 'value': res[prop], 'type': mytype}); } if(mytype=="object"){ rtnData.push({'name': prop, 'value': '', 'type': mytype}); } } return rtnData""" result = webdriver.execute_script(javascript) if len(result) > 0: for result_item in result: rtnData.append(result_item) except: print "error@get_application_classitem" pass return rtnData
def sendDate(id, txt): if (len(txt) > 0): splitDay = txt.split("/") gregorian_date = jdatetime.date(int(splitDay[0]), int(splitDay[1]), int(splitDay[2])).togregorian() day = webdriver.find_element_by_xpath( "/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div[" + str(id) + "]/span[1]/input[1]") month = webdriver.find_element_by_xpath( "/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div[" + str(id) + "]/span[1]/input[2]") year = webdriver.find_element_by_xpath( "/html/body/div/div/div/div/div/div/div/section[2]/form/fieldset/div/div/fieldset/div/div[1]/div[" + str(id) + "]/span[1]/input[3]") webdriver.execute_script( "arguments[0].setAttribute('value', '" + str(gregorian_date.day) + "')", day) webdriver.execute_script( "arguments[0].setAttribute('value', '" + str(gregorian_date.month - 1) + "')", month) webdriver.execute_script( "arguments[0].setAttribute('value', '" + str(gregorian_date.year) + "')", year)
def probe_DARLA_CONFIG(webdriver): try: darla = webdriver.execute_script("return DARLA_CONFIG.version") print "DARLA_CONFIG: "+darla except: pass
def probe_backbone(webdriver): try: backbone = webdriver.execute_script("return Backbone.VERSION") print "Backbone: "+backbone except: pass
f = open("/home/dhl/shipment_numbers.csv") wait = WebDriverWait(webdriver, 60 * 60) for line in f: #Searching for shipment code webdriver.find_element_by_id('shipmentCode').send_keys(line) webdriver.find_element_by_id('timeIntervall').send_keys('12') webdriver.find_element_by_name('search_ta').click() #Downloading pdf try: actionSelection = wait.until(EC.element_to_be_clickable((By.ID,'pageActionSelect'))) finally: pass actionSelection.send_keys('download') webdriver.execute_script('javascript:openLayerSelection(\'pieceList\',\'false\')') #webdriver.find_element_by_id('signchk').click() try: element = wait.until(EC.element_to_be_clickable((By.ID,'signchk'))) finally: pass element.click() webdriver.find_element_by_xpath("//input[@value='Download']").click() webdriver.implicitly_wait(600) webdriver.find_element_by_name('cancel_ta').click() print "downloaded shipment pdf with shipment code :" + line f.close()
def probe_drupal(webdriver): try: data = webdriver.execute_script("return Drupal") print "Drupal CMS detected" except: pass
def probe_modernizr(webdriver): try: modenizer = webdriver.execute_script("return Modernizr._version") print "Modernizr: "+modenizer except: pass
def probe_microsoft_dynamics_crm(webdriver): try: serverurl = webdriver.execute_script("return SERVER_URL") userlangcode = webdriver.execute_script("return USER_LANGUAGE_CODE") orglangcode = webdriver.execute_script("return ORG_LANGUAGE_CODE") orgname = webdriver.execute_script("return ORG_UNIQUE_NAME") appversion = webdriver.execute_script("return APPLICATION_FULL_VERSION") # if we got this far without an error then we must be crm print "Microsoft Dynamics CRM found:" print "\tSERVER_URL: "+serverurl print "\tUSER_LANGUAGE_CODE: "+str(userlangcode) print "\tORG_LANGUAGE_CODE: "+str(orglangcode) print "\tORG_UNIQUE_NAME: "+orgname print "\tAPPLICATION_FULL_VERSION: "+appversion #bonus print "\tIS_PATHBASEDURLS: "+str(webdriver.execute_script("return IS_PATHBASEDURLS")) print "\tIS_OUTLOOK_CLIENT: "+str(webdriver.execute_script("return IS_OUTLOOK_CLIENT")) print "\tIS_ONLINE: "+str(webdriver.execute_script("return IS_ONLINE")) print "\tIS_LIVE: "+str(webdriver.execute_script("return IS_LIVE")) print "\tIS_ONPREMISE: "+str(webdriver.execute_script("return IS_ONPREMISE")) print "\tCURRENT_WEB_THEME: "+str(webdriver.execute_script("return CURRENT_WEB_THEME")) print "" except: pass