def install_driver(): import pyderman as dr try: pyderman_path = dr.install(browser=dr.chrome, file_directory=sys._MEIPASS, verbose=True, chmod=True, overwrite=True, version=None, filename='chromedriver.exe', return_info=False) except: pyderman_path = dr.install(browser=dr.chrome, file_directory='./', verbose=True, chmod=True, overwrite=True, version=None, filename='chromedriver.exe', return_info=False) debugger.info('Installed chromedriver to path: %s' % pyderman_path) return pyderman_path
def start_browser(): opts = chromeOpts() opts.add_argument("--headless") opts.add_argument(f"user-agent={ua.random}") prefs = {'profile.managed_default_content_settings.images': 2} # Disallow images from loading opts.add_experimental_option("prefs", prefs) try: driver = Chrome(options=opts) except WebDriverException: print("Chromedriver not detected, it will now be downloaded...") install(browser=chrome, file_directory='./', filename="chromedriver.exe") driver = Chrome(options=opts) return driver
def open_chrome(headless=True, no_image=False, keep_log=True, allow_non_w3c=True, version="latest"): path = pyderman.install(browser=pyderman.chrome, version=version) options = Options() options.add_argument("--enable-javascript") options.headless = headless # Capabilities to keep logs. capabilities = DesiredCapabilities.CHROME if keep_log: capabilities["loggingPrefs"] = { "performance": "ALL" } # newer: goog:loggingPrefs # Allow non W3C standard command, see https://stackoverflow.com/q/56111529/610569 if allow_non_w3c: options.add_experimental_option('w3c', False) if no_image: prefs = {"profile.managed_default_content_settings.images": 2} options.add_experimental_option("prefs", prefs) driver = webdriver.Chrome(path, options=options, desired_capabilities=capabilities) # Sanity checks. driver.get("http://www.python.org") assert "Python" in driver.title return driver
def test_all_installs(self): for driver in all_drivers: print("Testing %s..." % driver.__name__) try: data = install(browser=driver, verbose=True, chmod=True, overwrite=True, return_info=True) except OSError as err: print(err) continue # OSError is raised if the given OS cannot support the driver, which we need to ignore. path = data['path'] if not os.path.exists(path): raise FileNotFoundError( 'The %s executable was not properly downloaded.' % driver.__name__) output = subprocess.check_output([path, '--version']).decode('utf-8') print('Version:', output) self.assertIn( data['version'], output.lower(), msg="Driver %s did not output proper version! ('%s')" % (driver.__name__, data['version'])) print('%s is installed at: "%s"' % (data['driver'], path)) print('\n\n\n')
def home(request): import pyderman as cdi from selenium.webdriver.common.touch_actions import TouchActions path = cdi.install(file_directory='c:\\data\\chromedriver\\', verbose=True, chmod=True, overwrite=False, version=None) from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait options = webdriver.ChromeOptions() options.add_argument("--ignore-certificate-errors") #options.add_argument("headless") driver = webdriver.Chrome("c:\\data\\chromedriver\\chromedriver.exe", chrome_options=options) driver.implicitly_wait(100) #driver.set_page_load_timeout(2) driver.get("https://www.amazon.com/gp/profile/amzn1.account.AG7BVCPAJAA55B2TAVXEEKUDBSVA?preview=true") #data = driver.find_element_by_class_name('a-profile-content') scroll(driver, 30) # Once scroll returns bs4 parsers the page_source soup = BeautifulSoup(driver.page_source, 'lxml') #lxml is faster than html # Them we close the driver as soup_a is storing the page source driver.close() # Empty array to store the links links = [] # Looping through all the a elements in the page source for link in soup.find_all('a'): link.get('href') links.append(link.get('href')) return render(request, 'gohere.html', {'done': links})
def install_browser(file_dir): path = driver.install(browser=driver.chrome, file_directory=file_dir, chmod=True, overwrite=False, verbose=True, version=None, filename='chromedriver')
def install_driver(): """ This method installs a chrome driver in the /src/lib directory to allow for websraping through Google Chrome. It returns the path to the chromedriver. """ PATH = driver.install(browser=driver.chrome) print("INSTALLED CHROMEDRIVER AT PATH: %s" % PATH) return PATH
def installDriver(): # check if the Chromedriver is installed and, if not, downloads everything required path = dr.install(browser=dr.chrome, file_directory='./lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) print('Installed chromedriver to path: %s' % path) return path
def install_browser_driver(args): if args.browser.lower() == 'firefox': path = dr.install(browser=dr.firefox, file_directory='./lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) elif args.browser.lower() == 'chrome': path = dr.install(browser=dr.chrome, file_directory='./lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) else: raise InvalidBrowserException( f"{args.browser} is not an expected browser") return path
def init_scraper(): cdi_path = Path('./chromedriver/') cdi_install_path = pyderman.install(file_directory=cdi_path, verbose=True, chmod=True, overwrite=False, version=None) chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("headless") chrome_options.add_argument("--log-level=3") return webdriver.Chrome(executable_path=cdi_install_path, options=chrome_options)
def load_driver(chrome_version): """ Loads up google chrome with selenium :return: driver object """ # Automatically download the chrome driver path = dr.install(browser=dr.chrome, version=chrome_version, file_directory='./Lib/') chrome_options = Options() chrome_options.add_argument("start-maximized") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") driver = webdriver.Chrome(path, options=chrome_options) return driver
def run(): for driver in [chrome, firefox]: print("Testing %s..." % driver.__name__) data = install(browser=driver, verbose=True, chmod=True, overwrite=True, return_info=True) path = data['path'] if not os.path.exists(path): raise FileNotFoundError( 'The %s executable was not properly downloaded.' % driver.__name__) output = subprocess.check_output([path, '--version']).decode('utf-8') print('Version:', output) assert data['version'] in output.lower() print('%s is installed at: "%s"' % (data['driver'], path)) print('\n\n\n')
import pyderman as dr path = dr.install(browser=dr.chrome, file_directory='./lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) print('Installed chromedriver to path: %s' % path)
from selenium import webdriver from selenium.webdriver.chrome.options import Options import time import pyderman #Install chrome driver driver_path = pyderman.install(browser=pyderman.chrome, overwrite=True, verbose=False) options = Options() options.headless = True with webdriver.Chrome(executable_path=driver_path, options=options) as driver: url = "http://lg.novoserve.com/rerouteintel.php" driver.get(url) passed = False for _ in range(15): if b"The results are in!" in driver.page_source.encode("utf-8"): passed = True break else: time.sleep(60) if not passed: raise StopIteration("Failed to set routes")
def get_identity_info(url, email, password, first_name="Bau1", last_name="EESADMIN", driver=None) -> (str, str): os.environ["PATH"] += os.pathsep + os.getcwd() + os.sep + 'webdriver' using_existing_driver = driver is not None if not driver: pyderman.install(file_directory="../webdriver/", filename='chromedriver', verbose=False, chmod=True, overwrite=False, version="latest") chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--headless') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--ignore-certificate-errors') driver = webdriver.Chrome(options=chrome_options) try: driver.get(url) wait_until_page_contains_xpath(driver, '//button[contains(text(), "Sign in")]') driver.find_element_by_xpath('//button[contains(text(), "Sign in")]').click() try: wait_until_page_contains_xpath(driver, '//div[text()="Sign in"]') time.sleep(1) except: raise AssertionError('Sign in page didn\'t appear?') try: driver.find_element_by_xpath('//input[@type="email"]').send_keys(email) time.sleep(1) driver.find_element_by_xpath('//input[@value="Next"]').click() wait_until_page_contains_xpath(driver, '//div[text()="Enter password"]') time.sleep(1) except: raise AssertionError('Error when entering/submitting email!') try: driver.find_element_by_xpath('//input[@type="password"]').send_keys(password) time.sleep(1) driver.find_element_by_xpath('//input[@value="Sign in"]').click() wait_until_page_contains_xpath(driver, '//div[text()="Stay signed in?"]') wait_until_page_contains_xpath(driver, '//input[@value="No"]') except: raise AssertionError('Error when entering/submitting password') time.sleep(1) driver.find_element_by_xpath('//input[@value="No"]').click() # Register user if necessary try: wait_until_page_contains_xpath(driver, '//span[contains(text(),"Register")]') driver.find_element_by_css_selector('#Input_FirstName').clear() driver.find_element_by_css_selector('#Input_FirstName').send_keys(first_name) driver.find_element_by_css_selector('#Input_LastName').clear() driver.find_element_by_css_selector('#Input_LastName').send_keys(last_name) driver.find_element_by_css_selector('#Input_Email').clear() driver.find_element_by_css_selector('#Input_Email').send_keys(email) driver.find_element_by_xpath('//button[contains(text(), "Register")]').click() except Exception: pass try: wait_until_page_contains_xpath(driver, '//h1[text()="Dashboard"]') # Should be Admin dashboard for user except: raise AssertionError( f'Couldn\'t find \'//h1[text()="Dashboard"]\' on page. Incorrect user details used? Found page source: \n{driver.page_source}') local_storage_json = driver.execute_script( f"return window.localStorage.getItem('GovUk.Education.ExploreEducationStatistics.Adminuser:{url}:GovUk.Education.ExploreEducationStatistics.Admin')") assert local_storage_json is not None, f"Couldn't find 'GovUk.Education.ExploreEducationStatistics.Adminuser:{url}:GovUk.Education.ExploreEducationStatistics.Admin' in Local Storage!" identity_cookie_dict = driver.get_cookie('.AspNetCore.Identity.Application') assert identity_cookie_dict is not None, "Couldn't get cookie '.AspNetCore.Identity.Application'" identity_cookie = json.dumps(identity_cookie_dict) return local_storage_json, identity_cookie finally: if not using_existing_driver: driver.close()
def create_driver(self, proxy=None, headless=True): if self.virtual: return if proxy and not isinstance(proxy, str): with open(os.path.join(os.getcwd(), 'tested_proxies.txt'), encoding="utf-8") as file: lines = file.read().split() proxy = random.choice(lines).strip() choice = random.choice([ ['chrome', dr.chrome], ['chrome', dr.chrome], ['chrome', dr.chrome], # ['firefox', dr.firefox], # ['opera', dr.opera], # dr.phantomjs ]) path = dr.install(browser=choice[1], file_directory='src/lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) if choice[0] == 'chrome': options = CO() if proxy: webdriver.DesiredCapabilities.CHROME['proxy'] = { "httpProxy": proxy, "ftpProxy": proxy, "sslProxy": proxy, "proxyType": "MANUAL", } elif choice[0] == 'firefox': options = FO() if proxy: webdriver.DesiredCapabilities.FIREFOX['proxy'] = { "httpProxy": proxy, "ftpProxy": proxy, "sslProxy": proxy, "proxyType": "MANUAL", } elif choice[0] == 'opera': options = OO() if proxy: webdriver.DesiredCapabilities.OPERA['proxy'] = { "httpProxy": proxy, "ftpProxy": proxy, "sslProxy": proxy, "proxyType": "MANUAL", } # opera_profile = '/Users/antonkurenkov/Proj/pbot/archive' # opera_profile = ' /Users/antonkurenkov/Library/Application Support/com.operasoftware.Opera' # options.add_argument('user-data-dir=' + opera_profile) options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--disable-gpu') options.add_argument("--disable-infobars") options.add_argument("--disable-extensions") # options.add_argument('--remote-debugging-port=9222') # options.add_argument("--disable-setuid-sandbox") # options.add_experimental_option('useAutomationExtension', False) # options.add_experimental_option('excludeSwitches', ['enable-logging']) if random.randint(0, 100) >= 30: # random device; mostly pc self.agent = self.create_useragent() options.add_argument(f'--user-agent={self.agent}') if random.randint(0, 100) >= 30: options.add_argument('--start-maximized') elif random.randint(0, 100) >= 30: options.add_argument("window-size=1920,1080") elif random.randint(0, 100) >= 30: options.add_argument("window-size=1024,768") else: # mobile device dims = ((360, 640), (375, 667), (414, 896), (360, 780), (360, 760), (375, 812), (360, 720), (414, 736), (412, 846), (360, 740), (412, 892), (412, 869), (393, 851), (412, 732), (320, 568), (720, 1280), (1080, 1920), (360, 800), (320, 570), (1080, 2340)) size = random.choice(dims) options.add_argument(f"window-size={size[0]},{size[1]}") with open('agents_m', encoding="utf-8") as file: self.agent = random.choice(file.read().split('\n')) options.add_argument(f'--user-agent={self.agent}') options.headless = headless if choice[0] == 'chrome': self.driver = webdriver.Chrome(options=options, executable_path=path) elif choice[0] == 'firefox': self.driver = webdriver.Firefox(options=options, executable_path=path) elif choice[0] == 'opera': self.driver = webdriver.Opera(options=options, executable_path=path)
# ====================================== print() print() # ~~~~~Instantiate Web Driver~~~~~ # NOTE the user must ensure that this directory is available, or edit it to one that is # checks to see if the Chromedriver is installed and, if not, downloads everything required. driverDir = 'C:\\Data\\chromedriver\\' import pyderman as driver path = driver.install(browser=driver.chrome, file_directory=driverDir, verbose=True, chmod=True, overwrite=False, version=None) print('Installed chromedriver to path: %s' % path) from selenium import webdriver from selenium.webdriver.chrome.options import Options # from selenium.webdriver.common.keys import Keys options = Options() options.headless = True # change this to change if browser window opens or not options.add_argument( '--log-level=1' ) # used to suppress extraneous messages. Defualt value is 0 (INFO)
def install_driver(): import pyderman pyderman.install(browser=pyderman.chrome, file_directory='lib/driver/', filename='chromedriver')
def get_proxy(): with open(os.path.join(os.getcwd(), 'proxies.txt'), encoding="utf-8") as file: pool = file.read().split() random.shuffle(pool) os.remove(os.path.join(os.getcwd(), 'proxies.txt')) for p in pool: # proxies = { # "http": p, # "https": p, # } try: path = ddd.install(browser=ddd.chrome, file_directory='src/lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) options = Options() webdriver.DesiredCapabilities.CHROME['proxy'] = { "httpProxy": p, "ftpProxy": p, "sslProxy": p, "proxyType": "MANUAL", } options.headless = False options.add_argument('--start-maximized') options.add_argument("--disable-extensions") options.add_argument("--disable-infobars") options.add_argument('--disable-gpu') options.add_argument('--disable-dev-shm-usage') options.add_argument('--no-sandbox') dr = webdriver.Chrome(executable_path=path, options=options) dr.get( 'https://www.payqrcode.ru/?&Ffbclid=DIwAR2ofk1GPgjyscpitWKkIBdY_7i6KE6V0ybX3DpY3GTh0qG9t3Bwkfjn3jw' ) ('&h=AT3Vv6So3ncpmvMOwc4HHyb2WAh7kYRovYWt8qvdQKStFDd4oF-LQFWWAUaFHZhEU3GtKv8wOURioVlFhs4j_kW6sfvRW1xfaTtiTnS5HIIRZQPSUmF18YoqflxG2wNtv1-NXBHMAw' + '&__tn__=-UK-R' + '&c[0]=AT22LLR7aEYoGYL-IlqGhRKfpGGKtR86lyqhhZ8o9C1II0YLuf2XYW0mOH685hUFXPlnWINKjB2axV4xCvxG8iFzN6_Bi5ETzuZUDuXvNPos1Oz6r9UFszylAItm9-L0zp0E' ) raw_html = WebDriverWait(dr, 10).until( EC.presence_of_element_located((By.XPATH, '//body'))).text if dr.title == 'Payment QR-code generator' and 'Block 1' in raw_html: with open(os.path.join(os.getcwd(), 'proxies.txt'), 'a+', encoding="utf-8") as file: file.write(p) file.write('\n') time.sleep(5) dr.get('http://2ip.ru') time.sleep(10) dr.quit() return p except Exception as e: print(e) dr.quit() raise e
print( '\n1. Aktualizacja drivera. \n\n2. Parser opon. \n21. Opony do csv. ' + '\n\n3. Parser motocykle. \n31. Motocykle do csv. \n\n4. Parser felg. \n41. Felgi do csv' + ' \n\n5. Parser alufelgi \n51 Alufelgi do csv') number = input('Wybierz numer: ') if number == '1': print('Aktualizacja drivera') # chd = ChromeDriverManager() # chd.download_and_install() # path = os.path.join(os.getcwd(), 'driver') # call(['webdrivermanager', 'chrome', f'-d {str(path)}']) # print(s) path = dr.install(browser=dr.chrome, file_directory=os.getcwd(), overwrite=True, filename='chromedriver') print(f'Zainstalowano : ${path}') print('Kopiowanie do C:\Windows') shutil.copy2(path, 'C:\Windows') elif number == '2': print('\n PARSER OPON \n') start_time = time.time() print(f'Czas rozpoczęcia: {start_time}') path = os.path.join(os.getcwd(), 'opony') os.chdir(path) opony_app() end_time = time.time() - start_time print(f'Trwało: {datetime.timedelta(seconds=end_time)}') elif number == '21':
last_height = new_height html.send_keys(Keys.HOME) #function to select each baanSize to load JSON URL def clickBaan(driver, baanSize): allClick = driver.find_elements_by_xpath("//*[contains(text()," + baanSize + ')]') for gonnaClick in allClick: gonnaClick.click() scrollAll(driver) return #Init driver path = pyder.install(browser=pyder.chrome, file_directory='./lib/') options = Options() options.add_argument("--headless") options.add_argument( "window-size=1920,1080") #use this size to avoid mobile window driver = webdriver.Chrome(options=options, executable_path=path) print(driver.get_window_size()) url = 'https://rubnongkaomai.com' driver.get(url) #Get BAAN to click (After observation, there is only 1 element with text() = 'BAAN') allClick = driver.find_elements_by_xpath("//*[text()='BAAN']") time.sleep(1) print(len(allClick)) #Click BAAN to get to BAAN page
if args.slack_webhook_url: os.environ['SLACK_WEBHOOK_URL'] = args.slack_webhook_url if args.admin_pass: os.environ['ADMIN_PASSWORD'] = args.admin_pass if args.analyst_pass: os.environ['ANALYST_PASSWORD'] = args.analyst_pass # Install chromedriver and add it to PATH chromedriver_filename = 'chromedriver.exe' if platform.system( ) == "Windows" else 'chromedriver' pyderman.install(file_directory='./webdriver/', filename=chromedriver_filename, verbose=True, chmod=True, overwrite=False, version=args.chromedriver_version) os.environ["PATH"] += os.pathsep + str(Path('webdriver').absolute()) output_file = "rerun.xml" if args.rerun_failed_tests or args.rerun_failed_suites else "output.xml" # Set robotArgs robotArgs = [ "--outputdir", "test-results/", "--output", output_file, "--exclude", "Failing", "--exclude", "UnderConstruction", "--exclude", "BootstrapData" ] robotArgs += [ "-v", f"timeout:{os.getenv('TIMEOUT')}", "-v",
#%% import pyderman as driver path = driver.install(browser=driver.chrome) print('Installed geckodriver driver to path: %s' % path) #%% from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.chrome.options import Options from bs4 import BeautifulSoup import lxml #%%launch url chrome_options = Options() #chrome_options.add_argument("--disable-extensions") #chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--headless") url = "https://shopee.co.id/Jeju-Natural-Mask-WELCOS--i.101515897.1715099836" #%% create a new Firefox session driver = webdriver.Chrome(path, options=chrome_options) driver.implicitly_wait(30) driver.get(url) #%% python_button = driver.find_elements_by_xpath("//div[@id='main']/div/div[2]/div[2]/div[2]/div[2]/div[3]/div/div[4]/div/div[2]/div/div/div/button[2]")[0] #python_button = driver.find_element_by_id('product-variation') #FHSU python_button.click() #click fhsu link soup=BeautifulSoup(driver.page_source, 'html.parser') page = soup.find("div", {"class": "flex items-center _1FzU2Y"}).getText()
def get_proxies_from_free_cz(): path = ddd.install(browser=ddd.chrome, file_directory='src/lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) options = Options() options.headless = True options.add_argument('--start-maximized') options.add_argument("--disable-extensions") options.add_argument("--disable-infobars") options.add_argument('--disable-gpu') options.add_argument('--disable-dev-shm-usage') options.add_argument('--no-sandbox') dr = webdriver.Chrome(executable_path=path, options=options) for i in [ '', # '2', # '3' ]: url = f'http://free-proxy.cz/ru/proxylist/country/RU/all/ping/all/{i}' dr.get(url) time.sleep(1) # hosts = [h.text for h in dr.find_elements_by_xpath('//table[@id="proxy_list"]/tbody/tr/td[1]') if h.text != ''] # ports = [p.text for p in dr.find_elements_by_xpath('//tbody/tr/td[2]')] hosts = [ h.text for h in WebDriverWait(dr, 10).until( EC.presence_of_all_elements_located(( By.XPATH, '//table[@id="proxy_list"]/tbody/tr/td[1]'))) if h.text != '' ] ports = [ p.text for p in WebDriverWait(dr, 10).until( EC.presence_of_all_elements_located((By.XPATH, '//tbody/tr/td[2]'))) ] ru_pool = [ f'http://{i[0]}:{i[1]}' for i in list(zip(hosts, ports)) ] if os.path.isfile(os.path.join(os.getcwd(), 'proxies.txt')): with open(os.path.join(os.getcwd(), 'proxies.txt'), 'r', encoding="utf-8") as file: poool = file.read().split() ru_pool += poool random.shuffle(ru_pool) with open(os.path.join(os.getcwd(), 'proxies.txt'), 'w+', encoding="utf-8") as file: file.write('\n'.join(ru_pool)) file.write('\n') dr.quit()
def before_all(context): context.path = driver.install(browser=driver.chrome)
def __init__(self, category, URL): self.path = dr.install(browser=dr.chrome, file_directory='./lib/', verbose=True, chmod=True, overwrite=False, version=None, filename=None, return_info=False) self.category = category self.URL = URL
import base64 from selenium import webdriver import pyderman as driver import time # Install webdriver path = driver.install(browser=driver.firefox, overwrite=False, verbose=True) # browser = webdriver.Firefox(executable_path=path) # # browser.get("https://web.whatsapp.com/") # # # # image_path = '/opt/whatsapp/auth/download.jpeg' # browser.get(image_path) # # driver = webdriver.Chrome() # browser.get("http://curran.github.io/HTML5Examples/canvas/smileyFace.html") browser.get("https://web.whatsapp.com/") time.sleep(5) image = browser.find_element_by_tag_name("canvas").screenshot_as_base64 # decode canvas_png = base64.b64decode(image) # save to a file with open(r"/tmp/canvas.png", 'wb') as f:
def download_chromedriver(): path = driver.install(browser=driver.chrome, file_directory=WORKSPACE,filename='chromedriver') print('Installed chromdriver driver to path: %s' % path)