def __init__(self, path=None, arabic=True): self.arabic = arabic # Browsing and writing managers self.br = Browser() self.wr = Writer(path) if path else Writer() # An array for scrapped books self._books_ids = []
def setupDock(self): # 建立输出运行结果的窗口 self.rightBrowser = Browser() # 建立在窗口右端 self.rightDock = QDockWidget('运行结果/内存', self) self.rightDock.setFeatures(QDockWidget.DockWidgetClosable) self.rightDock.setWidget(self.rightBrowser) self.addDockWidget(Qt.RightDockWidgetArea, self.rightDock) self.rightDock.hide() # 窗口自动隐藏 # 建立输出调试过程的窗口 self.bottomBrowser = Browser() # 建立在窗口底端 self.bottomDock = QDockWidget('调试窗口', self) self.bottomDock.setFeatures(QDockWidget.DockWidgetClosable) self.bottomDock.setWidget(self.bottomBrowser) self.addDockWidget(Qt.BottomDockWidgetArea, self.bottomDock) self.bottomDock.hide()
def createBrowser(self): self.listWidget.clear() if ((self.keywordsName != '') & (self.docName != '')): self.flag = 1 print(self.checkBox.isChecked()) self.browser = Browser(self.keywordsName, self.docName, self.checkBox.isChecked())
def setUp(self): self.driver = Browser().getbrowser("chrome") self.driver.get(self.url.webViewerUAT) self.driverUtility = DriverUtility(self.driver, self.log) self.loginPageObject = LoginPageObject(self.driverUtility, self.log) yield self.driver.close()
def crawl_nexon_board(): raw_data = [] browser = Browser() browser.get_page(URL.BASE_URL) browser.click_element_by_xpath(ElementXpath.POST_TITLES_ON_BOARD) recent_post_id = int( re.search('n4articlesn=(\d+)', browser.get_current_url()).group(1)) for i in range(0, 10): url = PathUtils.fromTemplate(URL.POST_URL, str(recent_post_id - i)) browser.get_page(url) title = browser.get_element_by_xpath( ElementXpath.TITLE_ON_POST).text content = browser.get_element_by_xpath( ElementXpath.CONTENT_ON_POST).get_attribute("content") comments = list( map( lambda element: element.text, browser.get_elements_by_xpath( ElementXpath.COMMENTS_ON_POST))) raw_data.append(title) raw_data.append(content) raw_data.extend(comments) return raw_data
def __init__(self, *args, obj=None, **kwargs): super().__init__(*args, **kwargs) self.browser = Browser() self.setupUi(self) self.load_config() self.set_ui() self.set_signals()
def __init__(self) -> None: self.offices = [] self.agents = [] self.agentNames = [] self.agentPositions = [] self.agentNumbers_1 = [] self.agentNumbers_2 = [] self.browser = Browser()
def create_widgets(self): self.browser = Browser() self.create_butonn_1() self.create_button_2() self.layout.addWidget(self.browser, 1, 1) self.layout.addWidget(self.button1, 2, 1) self.layout.addWidget(self.button2, 2, 2)
def __init__(self) -> None: self.officeNames = [] self.officeAddress = [] self.officeProvinces = [] self.phoneNumbers = [] self.websiteLinks = [] self.emails = [] self.agents = [] self.browser = Browser()
def add_new_tab(self, qurl, label): # Instead of browser it should be called WebView ! browser = Browser() browser.urlChanged.connect( lambda qurl, browser=browser: self.update_url_bar(qurl, browser)) indexTab = self.tabs.addTab(browser, label) # We need to update the url ! self.tabs.currentWidget().setUrl(QUrl(qurl)) return indexTab
def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) return req
def __init__(self, path=None, lang="ar", edition_reviews=False): # Language of reviews to be scraped self._lang = lang # Instantiate browsing and writing managers self.wr = Writer(path) if path else Writer() self.br = Browser(edition_reviews) # Initialize an empty threads list self._threads = [] # Counter for reviews from different languages self._invalid = None
def getRequest(self, pluginName, cj=None): # TODO: mostly obsolete, browser could be removed completely req = Browser(self.bucket, self.getOptions()) if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) return req
def __init__(self, baselink, username=None, password=None): self.browser = Browser() self.baselink = str(baselink).strip("/") + "/" self.browser.open_page(self.baselink + 'login.aspx') try: if username == None: self.browser.sendClick("//input[@id='defaultuser']") else: self.__login(username, password) except: pass
def follow_new(self, event): url, target = self.whichurl(event) if url: app = self.context.app self['background'] = app.prefs.Get(STYLEGROUP, ATEMPPREF) url = self.context.get_baseurl(url) from Browser import Browser Browser(app.root, app).context.load(url) self['background'] = app.prefs.Get(STYLEGROUP, AHISTPREF) else: self.context.viewer.leave_message()
def anchor_click_new(self, event): here = self.text.index("@%d,%d" % (event.x, event.y)) if self.current_index != here: return url = self.find_tag_url() if url: url, target = self.split_target(url) self.master.update_idletasks() from Browser import Browser app = self.context.app b = Browser(app.root, app) b.context.load(self.context.get_baseurl(url)) self.remove_temp_tag(histify=1)
def directUpload(self, filename, private=False, test=False): if self.bot is None: self.bot = Browser().getBot() self.webbot = Bot(self.bot) self.bot.get(self.url) utils.randomTimeQuery() self.cookies = Cookies(self.bot) self.bot.refresh() try: file_input_element = self.webbot.getVideoUploadInput() except Exception as e: print(f"Error: {e}") print( "Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py" ) file_input_element = None exit() abs_path = os.path.join(os.getcwd(), filename) try: file_input_element.send_keys(abs_path) except StaleElementReferenceException as e: try: self.bot.implicitly_wait(5) file_input_element = self.webbot.getVideoUploadInput() file_input_element.send_keys(abs_path) except Exception as e: print( "Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py" ) exit() # We need to wait until it is uploaded and then clear input. self.addCaptions() utils.randomTimeQuery() if private: self.webbot.selectPrivateRadio() # private video selection utils.randomTimeQuery() else: """ self.webbot.selectPublicRadio() # public video selection utils.randomTimeQuery() """ pass if not test: self.webbot.uploadButtonClick() # upload button input("Press any button to exit")
def start_checking(): browser = Browser() clear() count = 0 reposition(0, 0) print(colorama.Fore.BLUE + INTRO + colorama.Fore.RESET) valid = False reset_screen() try: count = 0 while not valid: website, line = website_list.pop(0) website_list.append([website, line]) curtime = datetime.now() seconds_passed = int( (curtime - website.last_checked).total_seconds()) if seconds_passed < REFRESH_PERIOD: time.sleep(REFRESH_PERIOD - seconds_passed) else: count += 1 cur_price, in_stock, name, item_valid = browser.check_website( website) website.update_website(cur_price, in_stock, name) if item_valid: print(colorama.Fore.GREEN + str(website) + colorama.Fore.RESET) time.sleep(3) valid = True else: if count == len(website_list) * 3: reset_screen() count = 0 reposition(line, 0) print(colorama.Fore.RED + str(website) + colorama.Fore.RESET) except KeyboardInterrupt: try: browser.driver.close() time.sleep(1) except Exception as e: print(e)
def getRequest(self, pluginName, account=None, type="HTTP"): self.lock.acquire() if type == "XDCC": return XDCCRequest(proxies=self.getProxies()) req = Browser(self.bucket, self.getOptions()) if account: cj = self.getCookieJar(pluginName, account) req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) self.lock.release() return req
def add_website(url, site_type, max_price): browser = Browser() if url not in URL_MAP.map.values(): uuid = URL_MAP.add(url) website = Website(uuid, url=url, site_type=site_type, max_price=max_price) cur_price, in_stock, name, value = browser.check_website(website) website.update_website(cur_price, in_stock, name) created = True else: created = False browser.driver.close() return created
def _get(self, url, sub_dir): capture = Capture(sub_dir, self.conf) browser = Browser(url, sub_dir, self.conf) links = [] try: capture.run() sleep(3) html = browser.get() links = self._get_links(html) sleep(30) except WebDriverException: self._create_exception_file(traceback.format_exc(), sub_dir) except KeyboardInterrupt: self._create_exception_file(traceback.format_exc(), sub_dir) finally: capture.kill() browser.close() return links
def main(): # First, we are going to create a browser google_chrome_driver_path = '/home/hamid/Desktop/javad/research/VENV/JTest/chromedriver' google_chrome_object = Browser(google_chrome_driver_path) google_chrome = google_chrome_object.create_chrome_tab() # Then we are going to start our browser to crawl the webpage url_for_scrapping = 'https://marketplace.atlassian.com/search?category=Time%20tracking&product=jira' google_chrome.get(url_for_scrapping) # After that we are using the wait().until to make us sure that the webpage loads successfully try: wait_time = 5000 element = WebDriverWait(google_chrome, wait_time).until( EC.presence_of_element_located( (By.XPATH, "//h3[@class='sc-ghsgMZ giyMKw sc-kkbgRg hkWXqv']"))) javad_spider = Jscrapper(google_chrome) # Calculate the number of click on "More Results" button number_per_page = 24 search_result_count = javad_spider.search_result_count_finder() number_for_click_on_button = (search_result_count) // (number_per_page) for num in range(number_for_click_on_button): # Click on button for seeing all plugins javad_spider.more_result() # wait for loading the page successfully javad_spider.wait_for_loading() # Create CSV file javad_spider.csv_maker() # Start collecting all the plugins information plugins = javad_spider.plugin_finder() javad_spider.fetch_data(plugins) finally: google_chrome.quit()
def uploadVideo(self, video_dir, videoText, startTime=0, endTime=0, private=True, test=False, scheduled=False, schdate="", schtime=""): video_dir = self.downloadIfYoutubeURL(video_dir) if not video_dir: return if self.bot is None: self.bot = Browser().getBot() self.webbot = Bot(self.bot) self.userRequest["dir"] = video_dir self.checkFileExtensionValid() self.userRequest["cap"] = self.IO.getHashTagsFromFile() # Initiate bot if isn't already. self.bot.get(self.url) self.userRequest["vidTxt"] = videoText # Cookies loaded here. self.cookies = Cookies(self.bot) self.bot.refresh() # User now has logged on and can upload videos time.sleep(3) self.inputVideo(startTime, endTime) self.addCaptions() utils.randomTimeQuery() if private: self.webbot.selectPrivateRadio() # private video selection else: self.webbot.selectPublicRadio() # public video selection utils.randomTimeQuery() if not test: self.webbot.uploadButtonClick() # upload button input("Press any button to exit")
def add_new_tab(self, qurl, label): # Instead of browser it should be called WebView ! browser = Browser() browser.urlChanged.connect( lambda qurl, browser=browser: self.update_url_bar(qurl, browser)) indexTab = self.tabs.addTab(browser, label) self.tabs.setCurrentIndex(indexTab) # We need to update the url ! if qurl.startswith('zero://'): # ZeroNet protocol url_array = qurl.split('/') qurl = 'http://127.0.0.1:43110/' + url_array[2] elif qurl.startswith('http://'): # http protocol pass else: # Nothing mentionned qurl = 'http://127.0.0.1:43110/' + qurl self.tabs.currentWidget().setUrl(QUrl(qurl)) return indexTab
def getRequest(self, pluginName, account=None, type="HTTP", **kwargs): self.lock.acquire() options = self.getOptions() options.update(kwargs) # submit kwargs as additional options if type == "XDCC": req = XDCCRequest(self.bucket, options) else: req = Browser(self.bucket, options) if account: cj = self.getCookieJar(pluginName, account) req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) self.lock.release() return req
def setup(request, setUpClass): print("initiating chrome driverd") driver = Browser().getbrowser("chrome") url = URL() driver.get(url.webViewerUAT) utility = Utility() # utility.createLogFolder() log = open(utility.logpath + "/WV-00.txt", "a+") driverUtility = DriverUtility(driver, log) loginPageObject = LoginPageObject(driverUtility, log) request.cls.driver = driver request.cls.url1 = url request.cls.utility = utility request.cls.driverUtility = driverUtility request.cls.loginPageObject = loginPageObject print("setup ended") yield driver driver.close()
def crawl(self, **args): source = args['source'] ext = args['ext'] reply_time = args['reply_time'] br = Browser() page = br.open(self.baseurl) new_reply_time = reply_time while True: links = PageParser.parse(page, source) for i, link in enumerate(links): if reply_time < link.reply_time: if i is 0: new_reply_time = link.reply_time self.db.insert('\t'.join( [str(link), source, json.dumps(ext)])) else: return new_reply_time try: page = br.follow_link(text='后页>') except: Logger.info('finished!') break return new_reply_time
def lib(): return Browser()
def test_none(lib: Browser): assert lib.get_timeout(None) == 10000 assert Browser(timeout=timedelta(seconds=0.001)).timeout == 1
""" Method to save screenshot to a given path with the function name :param funcName: Function name :param defaultPath: is an optional paramter :return: None """ path = os.path.dirname(os.path.dirname(__file__)) path = path + "\\reports\\report_" + str( datetime.datetime.now().date()) + os.sep try: if not os.path.exists(path): print("path is not there creating...") os.makedirs(path) name = path + funcName + str( datetime.datetime.now().time()).replace(":", "_") + ".jpg" print(name) self._browser.get_screenshot_as_file(name) except Exception, e: print(e) if __name__ == "__main__": params = {"name": "Vinay"} myBrowser = Browser(params) myBrowser.go_to("http://google.com") Webaction = WebElementAction(myBrowser) Webaction.input_text("SearchButton", "Vinay") Webaction.submit_form("SearchButton") #Webaction.press_key("SearchButton","ENTER") time.sleep(3) myBrowser.quit()