def __init__(self, user_agent=None, wait_time=2, fullscreen=False, options=None, **kwargs): options = Options() if options is None else options if user_agent is not None: options.add_argument("--user-agent=" + user_agent) if fullscreen: options.add_argument('--kiosk') prefs = {"download": { "default_directory": _DOWNLOAD_PATH, "directory_upgrade": True, "extensions_to_open": "" }} options.add_experimental_option("prefs", prefs) self.driver = Chrome(chrome_options=options, **kwargs) self.element_class = WebDriverElement self._cookie_manager = CookieManager(self.driver) super(WebDriver, self).__init__(wait_time)
def __init__(self, scenario, download_directory=default_download_directory, app_host=None, app_port=None, executable_path=None, pages=None): # Contract must_be(download_directory, "download_directory", (type(None), str)) must_be(app_host, "app_host", (type(None), str)) must_be(app_port, "app_port", (type(None), Number)) must_be(executable_path, "executable_path", (type(None), str)) must_be(pages, "pages", (dict, type(None))) # TODO[TJ]: This should be implemented as part of the future contracts # library if pages is not None: for key, value in pages.items(): must_be(key, "pages key", str) must_be(value, "pages value", AppPage) # self.scenario = scenario if download_directory is not None: options = ChromeOptions() prefs = {"download.default_directory": download_directory} options.add_experimental_option('prefs', prefs) else: options = None if app_host is not None: self.app_host = app_host if app_port is not None: self.app_port = app_port if executable_path is not None: self.executable_path = executable_path self.pages = pages super(ChromeBrowser, self).__init__( executable_path=self.executable_path, chrome_options=options) register_exit(self.quit)
def __init__(self, options=None, user_agent=None, wait_time=2, fullscreen=False, incognito=False, headless=False, **kwargs): options = Options() if options is None else options options.add_argument("--disable-infobars") options.add_argument("--disable-notifications") if 'accepted_languages' in kwargs: options.add_experimental_option("prefs", {"intl.accept_languages": kwargs['accepted_languages']}) del kwargs['accepted_languages'] if user_agent is not None: options.add_argument("--user-agent=" + user_agent) if incognito: options.add_argument("--incognito") if fullscreen: options.add_argument('--kiosk') if headless: options.add_argument('--headless') options.add_argument('--disable-gpu') self.driver = Chrome(chrome_options=options, **kwargs) self.element_class = WebDriverElement self._cookie_manager = CookieManager(self.driver) super(WebDriver, self).__init__(wait_time)
def initiateChromeWithSize(width, height, pixelRatio=2.0): mobile_emulation = { "deviceMetrics": { "width": width, "height": height, "pixelRatio": pixelRatio }, "userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19" } chrome_options = Options() chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) return webdriver.Chrome(chrome_options=chrome_options)
def get_chrome_driver(self): opts = Options() if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts)
def setUp(self): #disble images chrome_options = Options() chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2}) self.driver = webdriver.Chrome(chrome_options=chrome_options) # options=[] # options.append('--load-images=false') # self.driver=webdriver.PhantomJS(executable_path='C:\\Users\\VUGON\\Desktop\\phantomjs-2.0.0-windows\\phantomjs-2.0.0-windows\\bin\\phantomjs.exe',service_args=options) #self.driver.implicitly_wait(30) self.base_url = "https://m.facebook.com/profile.php?v=info&id=100001150323526&nocollections=1"
def setUp(self): try: self.log = logger.Logger.get_logger() self.input = TestInputSingleton.input self.servers = self.input.servers self.browser = self.input.ui_conf['browser'] self.replica = self.input.param("replica", 1) self.case_number = self.input.param("case_number", 0) self.machine = self.input.ui_conf['server'] self.driver = None self.shell = RemoteMachineShellConnection(self.machine) #avoid clean up if the previous test has been tear down if not self.input.param("skip_cleanup", True) \ or self.case_number == 1: self.tearDown() self._log_start(self) self._kill_old_drivers() #thread for selenium server if not self._is_selenium_running(): self.log.info('start selenium') self._start_selenium_thread() self._wait_for_selenium_is_started() self.log.info('start selenium session') if self.browser == 'firefox': self.log.info("firefox") # self.driver = webdriver.Remote(desired_capabilities={ # "browserName": "firefox", # "platform": "MAC", # }) self.driver = webdriver.Remote(command_executor='http://{0}:{1}/wd/hub' .format(self.machine.ip, self.machine.port), desired_capabilities=DesiredCapabilities.FIREFOX) elif self.browser == 'chrome': self.log.info("chrome") local_capabilities = DesiredCapabilities.CHROME chrome_options = Options() chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) local_capabilities.update(chrome_options.to_capabilities()) self.log.info(self.machine.ip) self.log.info(self.machine.port) self.driver = webdriver.Remote(desired_capabilities={ "browserName": "chrome", "platform": "MAC", }) self.log.info('start selenium started') self.driver.get("http://127.0.0.1:8091") self.username = "******" self.password = "******" self.driver.maximize_window() except Exception as ex: self.input.test_params["stop-on-failure"] = True self.log.error("SETUP WAS FAILED. ALL TESTS WILL BE SKIPPED") self.fail(ex)
def get_chrome_driver(self): """Setup and return a Chrom[e|ium] browser for Selenium.""" opts = Options() absp = os.path.abspath if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") opts.add_extension(self.pb_ext_path) # will fail if ext can't be found if self.browser_bin: # otherwise will use webdriver's default binary print "Browser binary:", absp(self.browser_bin) opts.binary_location = self.browser_bin # set binary location # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts)
def __init__(self, username=None, password=None, nogui=False): if nogui: self.display = Display(visible=0, size=(800, 600)) self.display.start() chrome_options = Options() chrome_options.add_argument('--dns-prefetch-disable') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--lang=en-US') chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en-US'}) self.browser = webdriver.Chrome('./assets/chromedriver', chrome_options=chrome_options) self.browser.implicitly_wait(25) self.logFile = open('./logs/logFile.txt', 'a') self.logFile.write('Session started - %s\n' \ % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) self.username = username or environ.get('INSTA_USER') self.password = password or environ.get('INSTA_PW') self.nogui = nogui self.do_comment = False self.comment_percentage = 0 self.comments = ['Cool!', 'Nice!', 'Looks good!'] self.photo_comments = [] self.video_comments = [] self.followed = 0 self.follow_restrict = load_follow_restriction() self.follow_times = 1 self.do_follow = False self.follow_percentage = 0 self.dont_include = [] self.automatedFollowedPool = [] self.dont_like = ['sex', 'nsfw'] self.ignore_if_contains = [] self.ignore_users = [] self.use_clarifai = False self.clarifai_secret = None self.clarifai_id = None self.clarifai_img_tags = [] self.clarifai_full_match = False self.like_by_followers_upper_limit = 0 self.like_by_followers_lower_limit = 0 self.aborting = False
def get(browser, download_dir): """Returns the WebDriver for the specified browser. """ print("Getting Driver") # clear temp folder basics.remove_files(download_dir) # set firefox profile and return instance of HBPDriver if browser == "firefox": profile = ff_profile(download_dir) return HBPDriverFirefox(profile) elif browser == "chrome": prefs = { "profile.default_content_settings.popups": 0, "download.prompt_for_download": "false", "download.default_directory": download_dir, "download.extensions_to_open": "", } chrome_options = Options() chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--no-sandbox") #This make Chromium reachable chrome_options.add_argument("--no-default-browser-check") #Overrides default choices chrome_options.add_argument("--no-first-run") chrome_options.add_argument("--disable-default-apps") chrome_options.add_argument("--test-type") chrome_options.add_experimental_option("prefs", prefs) PROXY = "bbpproxy.epfl.ch:80" webdriver.DesiredCapabilities.CHROME['proxy'] = { "httpProxy": PROXY, "ftpProxy": PROXY, "sslProxy": PROXY, "noProxy": None, "proxyType": "MANUAL", "class": "org.openqa.selenium.Proxy", "autodetect": False } chrome_logpath = os.path.join(basics.dir_main(), "browser.log") driver = webdriver.Chrome(chrome_options=chrome_options, service_args=["--verbose", "--log-path=%s" % chrome_logpath]) return HBPDriver(driver) return HBPDriverChrome(chrome_options=chrome_options, service_args=["--verbose", "--log-path=%s" % chrome_logpath]) else: raise ValueError("Unsopported browser %s" % browser)
def setup_chromedriver(mode): # configuring chromedriver: # my screen res: 1366*768: # activate for verbose logging: # service_log_path = "chromedriver.log" # service_args = ['--verbose'] chrome_options = Options() if mode is 'primary': chrome_options.add_argument("--restore-last-session") chrome_options.add_argument("user-data-dir=master_profile") chrome_options.add_argument("--window-size={},{}".format(1000, 800)) chrome_options.add_argument("--window-position={},{}".format(1, 1)) elif mode is 'tertiary': chrome_options.add_argument("--window-size={},{}".format(200, 800)) chrome_options.add_argument("--window-position={},{}".format(1050, 1)) # get rid off: --ignore-certificate-errors command-line flag warning: chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) chrome_options.add_argument("--disable-infobars") # add command line flags chrome = Chrome(chrome_options=chrome_options, # service_args=service_args, # service_log_path=service_log_path ) # log in if not already: if 'readcube.com/session/new' in chrome.current_url and mode is 'primary': chrome.get('https://app.readcube.com/#/library/collection/~/items') # logging in via email / password for i in range(5): try: sleep(3) chrome.find_element_by_id('email').send_keys(user) chrome.find_element_by_id('password').send_keys(pw) chrome.find_element_by_class_name('submit-button').click() except Exception as e: sleep(1.5) # print e else: break # returns driver: return chrome
def chrome_manager(self): opts = Options() if self.on_travis: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") opts.add_extension(self.extension_path) opts.binary_location = self.browser_path opts.add_experimental_option("prefs", {"profile.block_third_party_cookies": False}) caps = DesiredCapabilities.CHROME.copy() driver = webdriver.Chrome(chrome_options=opts, desired_capabilities=caps) try: yield driver finally: driver.quit()
def start(self): if self.engine == 'chrome': from selenium.webdriver.chrome.options import Options options = Options() options.set_headless(headless=True) options.add_argument('--disable-gpu') options.add_argument('--log-level=3') options.add_argument('--mute-audio') options.add_experimental_option('prefs', { 'profile.managed_default_content_settings.images': 2, }) options.binary_location = self.binary_path driver = webdriver.Chrome( chrome_options=options, executable_path=self.driver_path, ) elif self.engine == 'firefox': from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.options import Options options = Options() options.set_headless(headless=True) options.add_argument('--disable-gpu') profile = webdriver.FirefoxProfile() profile.set_preference('permissions.default.image', 2) profile.set_preference('media.volume_scale', '0.0') profile.set_preference('media.autoplay.enabled', False) binary = FirefoxBinary(self.binary_path) driver = webdriver.Firefox( firefox_profile=profile, firefox_options=options, firefox_binary=binary, executable_path=self.driver_path, log_file=os.devnull, ) driver.set_window_size(1920, 1080) self.driver = driver
def set_selenium_local_session(self): """Starts local session for a selenium server. Default case scenario.""" if self.aborting: return self if self.use_firefox: if self.firefox_profile_path is not None: firefox_profile = webdriver.FirefoxProfile(self.firefox_profile_path) else: firefox_profile = webdriver.FirefoxProfile() # permissions.default.image = 2: Disable images load, this setting can improve pageload & save bandwidth firefox_profile.set_preference('permissions.default.image', 2) self.browser = webdriver.Firefox(firefox_profile=firefox_profile) else: chromedriver_location = './assets/chromedriver' chrome_options = Options() chrome_options.add_argument('--dns-prefetch-disable') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--lang=en-US') # managed_default_content_settings.images = 2: Disable images load, this setting can improve pageload & save bandwidth # default_content_setting_values.notifications = 2: Disable notifications # credentials_enable_service & password_manager_enabled = false: Ignore save password prompt from chrome # 'profile.managed_default_content_settings.images': 2, # 'profile.default_content_setting_values.notifications' : 2, # 'credentials_enable_service': False, # 'profile': { # 'password_manager_enabled': False # } chrome_prefs = { 'intl.accept_languages': 'en-US' } chrome_options.add_experimental_option('prefs', chrome_prefs) self.browser = webdriver.Chrome(chromedriver_location, chrome_options=chrome_options) self.browser.implicitly_wait(self.page_delay) self.logFile.write('Session started - %s\n' \ % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) return self
class Webpage: def __init__(self, url, browser='chrome'): self.driver = None self.url = url self.continue_scan = True if browser == 'chrome': self.launch_chrome() else: self.launch_firefox() self.driver.get(self.url) def launch_chrome(self): self.chrome_options = Options() self.chrome_options.add_argument('--disable-extensions') self.chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) self.driver = webdriver.Chrome(chrome_options=self.chrome_options) self.driver.maximize_window() def launch_firefox(self): self.driver = webdriver.Firefox() self.driver.maximize_window() @classmethod def open(cls, url, browser='chrome'): wbpage = Webpage(url, browser) return wbpage def wait_for_url_change(self, function_to_execute): url_before = self.driver.current_url function_to_execute() while not self.did_page_change(url_before, self.driver.current_url): time.sleep(5) def did_page_change(self, value_old, value_new): if value_old == value_new: return False else: return True def close(self): self.driver.close()
def init_chrome_driver(timeout=30): chrome_options = Options() chrome_options.add_argument("--disable-bundled-ppapi-flash") chrome_options.add_argument("--disable-plugins-discovery") chrome_options.add_argument("--disable-webaudio") chrome_options.add_argument("--mute-audio") #chrome_options.add_argument("--no-startup-window") prefs = {} prefs["plugins.plugins_disabled"] = ["Adobe Flash Player", "Shockwave Flash"] prefs["profile.managed_default_content_settings.images"] = 2 #prefs["profile.managed_default_content_settings.media_stream"] = 2 chrome_options.add_experimental_option("prefs", prefs) path = Config.value(SECTION_COMMON, "chromedriver_path") if path: log("Chromedriver path: %s" % path, INFO) driver = webdriver.Chrome(executable_path=path, chrome_options=chrome_options) else: driver = webdriver.Chrome(chrome_options=chrome_options) driver.wait = WebDriverWait(driver, timeout) return driver
def __init__(self, **kwargs): driver_path = kwargs.get('driver_path', 'chromedriver') headless = kwargs.get('headless', False) no_image = kwargs.get('no_image', False) user_agent = kwargs.get('user_agent', '') proxy = kwargs.get('proxy', '') home_page = kwargs.get('home_page', '') options = Options() if headless: options.add_argument('--headless') if no_image: option_load_img = {'profile.managed_default_content_settings.images': 2} options.add_experimental_option('prefs', option_load_img) if user_agent: options.add_argument(f'user-agent={user_agent}') if proxy: options.add_argument(f'--proxy-server=http://{proxy}') self.chrome = webdriver.Chrome(executable_path=driver_path, chrome_options=options) self.chrome.maximize_window() if home_page: self.chrome.get(home_page)
class HypothesisSeleniumDriver(): def __init__(self,browser,ext_path=None): self.ext_path = ext_path self.driver = None self.options = Options() if browser == 'firefox': self.driver = webdriver.Firefox() if browser == 'chrome': if self.ext_path is not None: self.options.add_extension(self.ext_path) self.options.add_experimental_option('excludeSwitches', ['test-type','ignore-certificate-errors']) # https://code.google.com/p/chromedriver/issues/detail?id=1081 self.driver = webdriver.Chrome(chrome_options=self.options) if browser == 'ie': self.driver = webdriver.Ie() self.drivername = self.driver.name #self.driver.implicitly_wait(HypothesisSeleniumTiming().very_long_wait) self.driver.set_page_load_timeout(HypothesisSeleniumTiming().wait_for_page_load_secs) self.driver.set_script_timeout(HypothesisSeleniumTiming().wait_for_script_secs)
def test(self): if sys.argv[1].__eq__('prod'): self.main_url = "https://blazemeter.com" elif sys.argv[1].__eq__('qa'): self.main_url = "https://wwwqa.blazemeter.com" elif sys.argv[1].__eq__('dev'): self.main_url = "https://wwwdev.blazemeter.com" else: self.main_url = "https://wwwqa.blazemeter.com" for i in range(2, len(sys.argv)): driver_param = sys.argv[i] if driver_param.__eq__("Chrome"): self.driver = webdriver.Chrome() self.driver.implicitly_wait(30) self.driver.set_window_size(1920, 1080) page_elements["/"] = ['#try_it_now>a', 'TRY IT NOW'] elif driver_param.__eq__("iPhone4"): mobile_emulation = {"deviceName": "Apple iPhone 4"} chrome_options = Options() chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) self.driver = webdriver.Chrome(chrome_options=chrome_options) self.driver.implicitly_wait(30) page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW'] elif driver_param.__eq__("iPhone5"): mobile_emulation = {"deviceName": "Apple iPhone 5"} chrome_options = Options() chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) self.driver = webdriver.Chrome(chrome_options=chrome_options) self.driver.implicitly_wait(30) page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW'] elif driver_param.__eq__("Firefox"): self.driver = webdriver.Firefox() self.driver.implicitly_wait(30) self.driver.set_window_size(1920, 1080) page_elements["/"] = ['#try_it_now>a', 'TRY IT NOW'] elif driver_param.__eq__("Nexus7"): mobile_emulation = {"deviceName": "Google Nexus 7"} chrome_options = Options() chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) self.driver = webdriver.Chrome(chrome_options=chrome_options) self.driver.implicitly_wait(30) page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW'] else: self.driver = webdriver.Firefox() self.driver.implicitly_wait(30) self.driver.set_window_size(1920, 1080) self.over_check(self.driver) self.driver.quit()
def runSelenium(context, browser_name, browser_dict, cpeLocation, cmdSwitches, op_sys, cache_state, hover=False): sparrow_controller = browser_dict.get('controller') cpe_ip = browser_dict.get('ip') is_windows = op_sys == 'windows' sparrow_controller.stopSparrow() # Cleanup leftover .dat files from --bb-on-beer; hopefully this is temporary, waiting on sparrow fix sparrow_controller.removeLeftoverBBs() if not sparrow_controller.startWebdriver(exe_loc=WEBDRIVER_LOC[op_sys], whitelist_ip=context.common.jenkinsIp): return False driver_options = Options() driver_options.binary_location = sparrow_controller.getSparrowFullPath(op_sys) # Account for browser cache state user_data_warm = sparrow_controller.getSparrowUserDataWarmPath(op_sys) if cache_state.lower() == 'warm': driver_options.add_argument('--user-data-dir='+user_data_warm) if not sparrow_controller.removeDir(dir_path=user_data_warm): return False # Initialize new user data dir remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller) load_url("http://www.google.com", remote) time.sleep(20) try: remote.quit() except Exception as e: logging.exception("Selenium exception caught quiting remote after iteration on cpe %s" % cpe_ip) with open(context.urlListFile) as f: context.sitelist = f.read().split('\n') for s in cmdSwitches: driver_options.add_argument(s) driver_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) # Values used later in analysis context.urlListLen = len(context.sitelist) context.numClicksTotal = context.listIterations * context.urlListLen context.numClicksPerSite = context.listIterations logging.info("running webdriver test on %s with sparrow switches: %s now.." % (cpe_ip, "; ".join(cmdSwitches))) logging.info("Running selenium webdriver on %s with sitelist: %s" % (cpe_ip, str(context.sitelist))) # k = url, v = beerID ; used to verify that the current beer is unique and new beer_status_dict = {} # initialize for site in context.sitelist: beer_status_dict[site] = None # Pageloads happen here site_count = 1 for i in range(context.listIterations): try: remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller) for site in context.sitelist: try: if not site: continue logging.info('Loading url: %s on cpe: %s' % (site, cpe_ip)) remote.switch_to_window(content_tab) if site[0:15] == 'sparrow://crash': res = load_url_and_crash(site, remote) if cache_state.lower() == 'warm': sparrow_controller.removeDir(dir_path=user_data_warm) return res elif hover: load_on_hover(site, remote) else: load_url(site, remote) logging.info('%s loaded successfully, number of sites loaded on %s: %s' % (site, cpe_ip, site_count)) site_count += 1 remote.switch_to_window(beerstatus_tab) # Wait up to 20 seconds for beer ack num_tries = 20 trys = 0 for i in range(num_tries): load_url("sparrow://beerstatus/", remote) if check_beer_status(site, beer_status_dict, remote): break trys += 1 time.sleep(1) if trys == num_tries: logging.info("No beer ack recieved for %s" % site) except Exception as e: logging.exception("Selenium exception caught during site visit on cpe %s" % cpe_ip) remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller) continue # Need to quit between iterations in both warm and cold case in order to clean out hint cache try: remote.quit() except Exception as e: logging.exception("Selenium exception caught quiting remote after iteration on cpe %s" % cpe_ip) continue except Exception as e: logging.exception("Selenium exception caught during iteration on cpe %s" % cpe_ip) continue sparrow_controller.stopWebdriver() sparrow_controller.stopSparrow() # Clean up if cache_state.lower() == 'warm': sparrow_controller.removeDir(dir_path=user_data_warm) return True
# Per https://sites.google.com/a/chromium.org/chromedriver/capabilities docs: prefs = { "savefile.default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads", "download.default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads", "download.directory_upgrade" : 1, "download.prompt_for_download" : 1, } prefs = { "savefile": { "default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads"}, "download": { "default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads", "directory_upgrade" : 1, "prompt_for_download": 1, }, } chrome_options.add_experimental_option("prefs", prefs) if False: browser = webdriver.Chrome(executable_path="/Users/vsh/Downloads/Selenium/chromedriver", chrome_options=chrome_options) # Most likely plan: # - Edit profile's Preferences to set the downloads directory, so I don't have to maintain that # aspect manually. # - Proceed "one file downloaded at a time. # - List all files in download directory, kick off file download, then wait until there's one new file. In-progress download files have .crdownload extensions. # - Find a way to be sure the download ended successfully. See http://stackoverflow.com/questions/559096/check-whether-a-pdf-file-is-valid-python. # - Pass the one new file to a function that will move and rename it to represent the statement's period. if False: SigninBoa(browser)
import sys import os import requests import sqlite3 import module.Database import module.Produce from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.support.ui import WebDriverWait from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.keys import Keys from selenium.webdriver.chrome.options import Options chrome_options = Options() chrome_options.add_argument('--proxy-server=http://127.0.0.1:8118') chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2, 'profile.default_content_setting_values.notifications': 2}) facebook_usernames = [""] facebook_password = "" #encodes reload(sys) sys.setdefaultencoding('utf8') #setup module.Database.createDatabase() conn = sqlite3.connect('facebook.db') conn.text_factory = str def loginFacebook(driver, fbun): driver.implicitly_wait(120) driver.get("https://www.facebook.com/")
from selenium.webdriver.common.keys import Keys # 键盘按键操作 from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait # 等待页面加载某些元素 from selenium.webdriver.chrome.options import Options from datetime import date, timedelta import time import random import json import re # 加启动配置 禁用日志log chrome_options = Options() # “–no - sandbox”参数是让Chrome在root权限下跑 chrome_options.add_argument('–no-sandbox') chrome_options.add_argument('–disable-dev-shm-usage') chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) chrome_options.add_argument('--start-maximized') # 最大化 chrome_options.add_argument('--incognito') # 无痕隐身模式 chrome_options.add_argument("disable-cache") # 禁用缓存 chrome_options.add_argument('log-level=3') chrome_options.add_argument('disable-infobars') chrome_options.add_argument('--headless') url = "https://newids.seu.edu.cn/authserver/login?service=http://ehall.seu.edu.cn/qljfwapp2/sys/lwReportEpidemicSeu/*default/index.do" enter_campus_url = "http://ehall.seu.edu.cn/ygfw/sys/swmxsqjappseuyangong/*default/index.do#/" # 申请入校网址 dailyDone1 = False # 创建打卡记录log文件 def writeLog(text):
from django.test import TestCase import requests import time # Create your tests here. from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys chrome_option = Options() chrome_option.add_argument('--disable-extensions') # 这里的端口和上面remote-debug对应的端口一致 chrome_option.add_experimental_option('debuggerAddress', '127.0.0.1:9222') # browser = webdriver.Chrome(executable_path='/Users/vium/chromedriver', chrome_options=chrome_option) options = webdriver.ChromeOptions() options.add_experimental_option('excludeSwitches', ['enable-automation']) browser = webdriver.Chrome(executable_path='/Users/vium/chromedriver', options=options, chrome_options=chrome_option) browser.get('http://www.nhc.gov.cn/xcs/xxgzbd/gzbd_index.shtml') content = browser.page_source # ul = browser.find_element_by_xpath("/html/body/div[3]/div[1]/ul") time.sleep(3) # session = requests.session() # session.headers.update( # { # 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36' # } # ) # r = session.get(url='http://www.nhc.gov.cn/xcs/xxgzbd/gzbd_index.shtml')
class Mydriver: def __init__(self, noimg=True, nohead=True): try: self.options = Options() if os.path.exists("./chrome/chrome.exe"): # win self.options.binary_location = "./chrome/chrome.exe" elif os.path.exists("/opt/google/chrome/chrome"): # linux self.options.binary_location = "/opt/google/chrome/chrome" if noimg: self.options.add_argument('blink-settings=imagesEnabled=false') # 不加载图片, 提升速度 if nohead: self.options.add_argument('--headless') self.options.add_argument('--disable-extensions') self.options.add_argument('--disable-gpu') self.options.add_argument('--no-sandbox') self.options.add_argument('--mute-audio') # 关闭声音 self.options.add_argument('--window-size=400,500') self.options.add_argument('--window-position=800,0') self.options.add_argument('--log-level=3') self.options.add_argument('--user-agent={}'.format(user_agent.getheaders())) self.options.add_experimental_option('excludeSwitches', ['enable-automation']) # 绕过js检测 self.webdriver = webdriver if os.path.exists("./chrome/chromedriver.exe"): # win self.driver = self.webdriver.Chrome(executable_path="./chrome/chromedriver.exe", chrome_options=self.options) elif os.path.exists("./chromedriver"): # linux self.driver = self.webdriver.Chrome(executable_path="./chromedriver", chrome_options=self.options) elif os.path.exists("/usr/lib64/chromium-browser/chromedriver"): # linux 包安装chromedriver self.driver = self.webdriver.Chrome(executable_path="/usr/lib64/chromium-browser/chromedriver", chrome_options=self.options) elif os.path.exists("/usr/local/bin/chromedriver"): # linux 包安装chromedriver self.driver = self.webdriver.Chrome(executable_path="/usr/local/bin/chromedriver", chrome_options=self.options) else: self.driver = self.webdriver.Chrome(chrome_options=self.options) #self.driver.set_window_size(1200, 900) except: print("=" * 120) print("Mydriver初始化失败") print("=" * 120) raise def login(self): print("正在打开二维码登陆界面,请稍后") self.driver.get("https://pc.xuexi.cn/points/login.html") try: remover = WebDriverWait(self.driver, 30, 0.2).until( lambda driver: driver.find_element_by_class_name("redflagbox")) except exceptions.TimeoutException: print("网络缓慢,请重试") else: self.driver.execute_script('arguments[0].remove()', remover) try: remover = WebDriverWait(self.driver, 30, 0.2).until( lambda driver: driver.find_element_by_class_name("header")) except exceptions.TimeoutException: print("当前网络缓慢...") else: self.driver.execute_script('arguments[0].remove()', remover) try: remover = WebDriverWait(self.driver, 30, 0.2).until( lambda driver: driver.find_element_by_class_name("footer")) except exceptions.TimeoutException: print("当前网络缓慢...") else: self.driver.execute_script('arguments[0].remove()', remover) self.driver.execute_script('window.scrollTo(document.body.scrollWidth/2 - 200 , 0)') try: WebDriverWait(self.driver, 270).until(EC.title_is(u"我的学习")) cookies = self.get_cookies() return cookies except: print("扫描二维码超时") def dd_login(self, d_name, pwd): __login_status = False self.driver.get( "https://login.dingtalk.com/login/index.htm?" "goto=https%3A%2F%2Foapi.dingtalk.com%2Fconnect%2Foauth2%2Fsns_authorize" "%3Fappid%3Ddingoankubyrfkttorhpou%26response_type%3Dcode%26scope%3Dsnsapi" "_login%26redirect_uri%3Dhttps%3A%2F%2Fpc-api.xuexi.cn%2Fopen%2Fapi%2Fsns%2Fcallback" ) self.driver.find_element_by_id("mobile").click() self.driver.find_element_by_id("mobile").send_keys(d_name) self.driver.find_element_by_id('pwd').click() self.driver.find_element_by_id("pwd").send_keys(pwd) self.driver.find_element_by_id("loginBtn").click() pic_src = self.driver.find_element_by_class_name("indentify_content").find_elements_by_tag_name('img')[0].get_attribute('src') if pic_src !='': print("遇到验证码!",pic_src) response = urllib.request.urlopen(pic_src) #print(response.status) #print(response.getheaders()) #print(response.read()) cat_b64 = str(base64.b64encode(response.read()))[2:-1] obj = self.send_captcha_ocr(cat_b64) if obj['code']==0: print('验证码识别结果为:' + obj['data']['captcha']) #self.driver.find_element_by_id("identifyCode").send_keys("123") #这个验证码框默认不可交互 self.go_js("document.querySelector('#identifyCode').value='" + obj['data']['captcha']+"'"); #print(self.driver.find_element_by_id("identifyCode").get_attribute('value')) self.driver.find_element_by_id("identifybtn").click() #提交按钮 else: print('验证码识别出错!') #self.go_js("document.querySelector('.indentify_content').scrollTo();") #screenImg = "/tmp/yzm.png" #self.driver.get_screenshot_as_file(screenImg) #屏幕截图 #with open(screenImg,"rb") as f: # imgdata = base64.b64encode(f.read()) # print(str(imgdata)) #sys.exit(0) try: print("登陆中...") WebDriverWait(self.driver, 30, 0.1).until(lambda driver: driver.find_element_by_class_name("modal")) self.quit() __login_status = False except: __login_status = True return __login_status def get_cookies(self): cookies = self.driver.get_cookies() return cookies def set_cookies(self, cookies): for cookie in cookies: self.driver.add_cookie({k: cookie[k] for k in cookie.keys()}) def get_url(self, url): self.driver.get(url) def go_js(self, js): self.driver.execute_script(js) def quit(self): self.driver.quit() def send_captcha_ocr(self,base64): url = '端口URL' dict = {'image': base64,'type': '1001','length':4,} data = urllib.parse.urlencode(dict).encode('utf-8') headers={'Authorization':'密钥','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'};#(请求headers头有一些供应商是key) request = urllib.request.Request(url = url,data = data,headers = headers,method = 'POST') response = urllib.request.urlopen(request) content = response.read() if (content): obj = json.loads(content) return obj
def navi (email, password): driver.get("https://open.spotify.com/playlist/7MQ2ATqqdDTnlx3UUDHe5d?si=d3bJe6odSe-b2Ww8obRR_A") time.sleep(3) bouton_login = driver.find_element_by_xpath('//*[@id="main"]/div/div[2]/div[1]/header/div[5]/button[2]') bouton_login.click() time.sleep(6) champ_email = driver.find_element(By.ID, "login-username") champ_email.send_keys(email + Keys.TAB) champ_password = driver.find_element(By.ID, "login-password") champ_password.send_keys(password + Keys.ENTER) time.sleep(5) bouton_play = driver.find_element_by_xpath('/html/body/div[4]/div/div[2]/div[4]/main/div/div[2]/div/div/div[2]/section/div[3]/div/button[1]') bouton_play.click() # mettre la boucle on click jsuque'à ce qu'on soit en mode repeat simple time.sleep(2) check_play() check_repeat() check_time() refresh() options = Options() options.add_experimental_option('useAutomationExtension', False) driver = webdriver.Chrome('/Users/aveyronvictor/Documents/AUTO/chromedriver', options=options) navi("*****@*****.**", "azerty123")
async def carbon_api(e): RED = random.randint(0, 256) GREEN = random.randint(0, 256) BLUE = random.randint(0, 256) THEME = [ "3024-night", "a11y-dark", "blackboard", "base16-dark", "base16-light", "cobalt", "dracula", "duotone-dark", "hopscotch", "lucario", "material", "monokai", "night-owl", "nord", "oceanic-next", "one-light", "one-dark", "panda-syntax", "paraiso-dark", "seti", "shades-of-purple", "solarized", "solarized%20light", "synthwave-84", "twilight", "verminal", "vscode", "yeti", "zenburn", ] CUNTHE = random.randint(0, len(THEME) - 1) The = THEME[CUNTHE] if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"): """ A Wrapper for carbon.now.sh """ await e.edit("⬜⬜⬜⬜⬜") CARBON = "https://carbon.now.sh/?bg=rgba({R}%2C{G}%2C.{B}%2C1)&t={T}&wt=none&l=auto&ds=false&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Fira%20Code&fs=14px&lh=152%25&si=false&es=2x&wm=false&code={code}" CARBONLANG = "en" textx = await e.get_reply_message() pcode = e.text if pcode[6:]: pcode = str(pcode[6:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded url = CARBON.format(code=code, R=RED, G=GREEN, B=BLUE, T=The, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = Config.GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {"download.default_directory": "./"} chrome_options.add_experimental_option("prefs", prefs) await e.edit("⬛⬛⬜⬜⬜") driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER, options=chrome_options) driver.get(url) download_path = "./" driver.command_executor._commands["send_command"] = ( "POST", "/session/$sessionId/chromium/send_command", ) params = { "cmd": "Page.setDownloadBehavior", "params": { "behavior": "allow", "downloadPath": download_path }, } driver.execute("send_command", params) driver.find_element_by_xpath( "//button[contains(text(),'Export')]").click() sleep(5) # this might take a bit. # driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() # sleep(5) await e.edit("⬛⬛⬛⬜⬜") # driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click() # sleep(5) #Waiting for downloading await e.edit("⬛⬛⬛⬛⬛") file = "./carbon.png" await e.edit("✅RGB Karbon Completed, Uploading RGB Karbon✅") await e.client.send_file( e.chat_id, file, caption= "✅Carbonised by [IndianBot](https://github.com/indianbhaiya/IndianBot/)", force_document=False, reply_to=e.message.reply_to_msg_id, ) os.remove("./carbon.png") # Removing carbon.png after uploading await e.delete() # Deleting msg
def _esperar(tempo): i = 0 while i < tempo: time.sleep(1) i += 1 # Lidar com notificações do Chrome option = Options() option.add_argument("--disable-infobars") option.add_argument("start-maximized") option.add_argument("--disable-extensions") # 1=aceitar / 2=recusar option.add_experimental_option( "prefs", {"profile.default_content_setting_values.notifications": 2}) # Abrir página browser = webdriver.Chrome( chrome_options=option, executable_path= "C:\\Users\\thith\Downloads\chromedriver_win32\chromedriver.exe") browser.maximize_window() browser.get('https://dashboard.tawk.to/login') email = "email" senha = "senha" browser.find_element_by_xpath( "/html/body/div[3]/div[2]/div/form/div[2]/input").send_keys(email) browser.find_element_by_xpath(
import cx_Oracle #Oracle import MySQLdb #MySQL import pymssql #MsSQL ----WebDriver---- driver = webdriver.Chrome(executable_path = "",chrome_options=) #To open a browser driver.get("http://newtours.demoaut.com/") #To open url ----Browser options---- from selenium.webdriver.chrome.options import Options chromeOptions=Options() chromeOptions.add_experimental_option("pref",{"download.default_directory":"C:\NewFolder"}) driver = webdriver.Chrome(chrome_options=chromeOptions) ----headless browser---- chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--window-size=1920x1080") print(driver.title) #To get the page title assert "Welcome: Mercury Tours" in driver.title #To verify the title print(driver.current_url) #To get the current url print(driver.page_source) #To get the HTML content of the page_source
########################################################################### Main Function ########################################################################### SUFFIX = {'A':'', 'S':'.S', 'Q':'.Q', 'M':'.M', 'W':'.W'} REPL = {'A':'', 'S':None, 'Q':'-Q', 'M':'-', 'W':None} new_item_counts = 0 chrome = None for g in range(start_file,last_file+1): if main_file.empty == False: break if chrome == None: options = Options() options.add_argument("--disable-notifications") options.add_argument("--disable-popup-blocking") options.add_argument("ignore-certificate-errors") options.add_experimental_option("excludeSwitches", ["enable-logging"]) chrome = webdriver.Chrome(ChromeDriverManager().install(), options=options) chrome.set_window_position(980,0) logging.info('Reading file: '+NAME+str(g)+' Time: '+str(int(time.time() - tStart))+' s'+'\n') if g == 1 or g == 2 or g == 8 or g == 9:############################################################ ECB ################################################################## file_path = data_path+NAME+str(g)+'.csv' if PRESENT(file_path): if g == 1 or g == 2: skip = [0,4] else: skip = [3] FOREX_t = readFile(data_path+NAME+str(g)+'.csv', header_ = [0,1,2], index_col_=0, skiprows_=skip) #FOREX_t = readFile(data_path+NAME+str(g)+'.csv', header_=[0], index_col_=0) else: if g == 1 or g == 8: url = 'https://sdw.ecb.europa.eu/browse.do?node=9691296'
import time from selenium.webdriver.chrome.options import Options from webdriver_manager.chrome import ChromeDriverManager import os from bs4 import BeautifulSoup from selenium.common.exceptions import NoSuchElementException from selenium.common.exceptions import ElementNotInteractableException from selenium.common.exceptions import ElementClickInterceptedException from selenium.common.exceptions import StaleElementReferenceException url = 'https://www.wsj.com/news/archive/2020/march' chrome_options = Options() chrome_options.add_argument("--window-size=1920,1080") prefs = {"profile.managed_default_content_settings.images": 2} chrome_options.add_experimental_option("prefs", prefs) driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options) # url_article = requests(url) driver.get(url) sign_in_link = driver.find_element_by_link_text('Sign In') sign_in_link.click() username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'username'))) password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'password'))) user1 = os.environ.get("USER") pass1 = os.environ.get("PASS")
from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC reload(sys) sys.setdefaultencoding("utf-8") DOMAIN = r"http://spiss.91safety.com/" PAGE = r"http://spiss.91safety.com/entOpen/firstpage.jhtml" CP = Options() CP.add_argument("--disable-extensions") CP.add_argument("--disable-popup-blocking") CP.add_experimental_option("prefs", { "credentials_enable_service": False, "profile": { "password_manager_enabled": False } }) ACCOUNT_PATH = "account2.xlsx" name = "" pw = "" res_name = u"" phone_number = "" department = u"办公室" cond = u"已经完成整改" pos = u"室内" service_args=[] service_args.append('--load-images=no') ##关闭图片加载 service_args.append('--disk-cache=yes') ##开启缓存
# * * # * June 01,2020 * # * * # * * * * * * * * * * * * * * * * * * * * * * * # import packages from selenium import webdriver from selenium.webdriver.chrome.options import Options import time from selenium.webdriver.common.keys import Keys options = Options() prefs = {'download.default_directory' : '/home/radia/lrcfiles'} options.add_experimental_option('prefs', prefs) driver = webdriver.Chrome(options=options) # Authentication url = "https://www.megalobiz.com/auth/login" driver.get(url) print('==== \n') print("Successfully, Started the Selenium Server! \n ") email = driver.find_element_by_css_selector('#email_macro_1') email.send_keys('*******@gmail.com') password = driver.find_element_by_css_selector('#password_macro_1') password.send_keys('*****') login = driver.find_element_by_css_selector('#setting_submit_form_macro_2') login.click() time.sleep(4)
def handle(self, *args, **kwargs): #Open chrome and navigate to webpage url = 'https://secure.sos.state.or.us/orestar/gotoPublicTransactionSearch.do' print('url collected') #Pass arguments to the Chrome Driver options = Options() prefs = {"download.default_directory": "./code/downloads"} options.add_experimental_option("prefs", prefs) options.add_argument('--disable-dev-shm-usage') options.add_argument('start-maximized') options.add_argument('disable-infobars') options.add_argument('--disable-extensions') options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') options.set_headless(headless=True) options.binary_location = "/usr/bin/google-chrome-stable" print('options collected') #Initiate Driver driver = webdriver.Chrome('/code/chromedriver', chrome_options=options) delay = 3 print('driver created') #Go to Orestar Transaction Search page driver.get(url) #Wait until page is loaded try: myElem = WebDriverWait(driver, delay).until( EC.presence_of_element_located((By.ID, 'header2'))) print("submitting search query") except TimeoutException: print("loading took too much time!") #Grab form and button elements start = driver.find_element_by_id('cneSearchTranStartDate') end = driver.find_element_by_id('cneSearchTranEndDate') search_button = driver.find_element_by_name('search') #Clear form input by default and input start and end dates start.clear() start.send_keys('06/15/2019') end.clear() end.send_keys('06/20/2019') #click search button search_button.click() print('searching specified dates') table = driver.find_element_by_tag_name('tr') next_btn = driver.find_element_by_name('next') transactions = [] #while the number of rows is greater than 1 find and click the next button to repeat the following steps while len(driver.find_elements_by_class_name('evenRow')) > 1: table = driver.find_element_by_tag_name('tr') even_rows = table.find_elements_by_class_name('evenRow') odd_rows = table.find_elements_by_class_name('oddRow') rows = even_rows + odd_rows #insert items into a list for items in rows: individual = [] odd_chars = ['$', '"'] #insert individual transactions into a list for cell in items.find_elements_by_xpath('.//td'): if odd_chars in cell: individual.append(cell.text) transactions.append(individual) try: wait_for_next = WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.NAME, "next"))) finally: next_btn = driver.find_element_by_name('next') next_btn.click() print('next page') else: # header = driver.find_element_by_class_name('shadedHeader') filename = time.strftime('%Y%m%d-%H%M%S') myFile = open('{}.csv'.format(filename), 'w') with myFile: writer = csv.writer(myFile) writer.writerow([ 'Trans ID', 'Original ID', 'Trans Date', 'Filer', 'Payee', 'SubType', 'Amount', 'Agg Amount', 'Payee ID', 'Filer ID', 'Attest by Name', 'Attest Date', 'Review by name', 'Review Date', 'Due Date', 'Occptn ltr date', 'Payment Text', 'Purps Desc', 'Interest', 'check_numb', 'trans_status_ind', 'File by Name', 'File Date', 'Address Book Agent', 'Book Type', 'Title Text', 'Occptn TXT', 'emp_name', 'emp_city', 'emp state', 'employeed', 'self employeed', 'Address line 1', 'Address line 2', 'city', 'state', 'zipcode', 'zipplus', 'county', 'purpose code' ]) writer.writerows(transactions) print('Records downloaded to {}.csv'.format(filename)) # //TODO: Write test coverage plan # //TODO: Rewrite scraper to download, rename and move the excel file #// TODO: Rewrite the loading script to match new fields and load from csv file #// TODO: Write script to download excel files a week at a time
def get_stock_data(stock_name, out_file_name, from_day, from_month, from_year, to_day, to_month, to_year): from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import Select from selenium.webdriver.support.ui import WebDriverWait as wait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC import time options = Options() options.add_experimental_option( "prefs", { "download.default_directory": r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks", "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True }) # url = "https://www.bseindia.com/markets/equity/EQReports/StockPrcHistori.aspx?expandable=7&scripcode=532648&flag=sp&Submit=G" url = "https://www.bseindia.com/markets/equity/EQReports/StockPrcHistori.aspx?scripcode=500247" chrome_driver = r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks\chromedriver.exe" driver = webdriver.Chrome(chrome_driver, options=options) driver.set_page_load_timeout(45) driver.get(url) time.sleep(1) driver.find_element_by_id("ContentPlaceHolder1_smartSearch").clear() time.sleep(1) driver.find_element_by_id("ContentPlaceHolder1_smartSearch").send_keys( stock_name) time.sleep(2) time.sleep(1) drop_down = driver.find_elements_by_css_selector("li.quotemenu a") for values in drop_down: values.click() break #driver.find_element_by_id("ContentPlaceHolder1_smartSearch").click() time.sleep(1) driver.find_element_by_id("ContentPlaceHolder1_txtFromDate").click() time.sleep(1) if int(from_year) < 2009: mySelect = Select( driver.find_element_by_class_name("ui-datepicker-year")) time.sleep(1) mySelect.select_by_visible_text("2009") time.sleep(1) mySelect = Select(driver.find_element_by_class_name("ui-datepicker-year")) time.sleep(1) mySelect.select_by_visible_text(from_year) mySelect = Select(driver.find_element_by_class_name("ui-datepicker-month")) mySelect.select_by_visible_text(from_month) wait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//td[@data-handler='selectDay']/a[text()='{}']".format(from_day) ))).click() time.sleep(1) driver.find_element_by_id("ContentPlaceHolder1_txtToDate").click() time.sleep(1) if int(to_year) < 2009: mySelect = Select( driver.find_element_by_class_name("ui-datepicker-year")) time.sleep(1) mySelect.select_by_visible_text("2009") time.sleep(1) mySelect = Select(driver.find_element_by_class_name("ui-datepicker-year")) time.sleep(1) mySelect.select_by_visible_text(to_year) time.sleep(1) mySelect = Select(driver.find_element_by_class_name("ui-datepicker-month")) mySelect.select_by_visible_text(to_month) wait(driver, 10).until( EC.visibility_of_element_located( (By.XPATH, "//td[@data-handler='selectDay']/a[text()='{}']".format(to_day) ))).click() time.sleep(1) driver.find_element_by_id('ContentPlaceHolder1_btnSubmit').click() time.sleep(2) driver.find_element_by_id('ContentPlaceHolder1_btnDownload').click() time.sleep(2) driver.quit() import os import shutil path = r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks" os.chdir(path) filename = max([f for f in os.listdir(path)], key=os.path.getctime) shutil.move(os.path.join(path, filename), out_file_name) return True
def open_class(link, runtime, class_name, record_class=False, debug_port=debug_port, exec_path=exec_path, user_data_dir=user_data_dir, exit_browser=True): if "https://" not in link: link = "https://" + link debug_address = "127.0.0.1:" + str(debug_port) cmd = "chrome --remote-debugging-port=" + str( debug_port) + " --user-data-dir=" + user_data_dir chrome_options = Options() chrome_options.add_experimental_option("debuggerAddress", debug_address) if mute: chrome_options.add_argument("--mute-audio") try: print("Trying to connect to driver") driver = webdriver.Chrome(options=chrome_options, executable_path=exec_path) print("Connected") except selenium.common.exceptions.WebDriverException as e: print("Couldnt connect\nTrying to open") if 'cannot connect to chrome at' in str(e): process = subprocess.Popen(cmd) print("Trying to connect") driver = webdriver.Chrome(options=chrome_options, executable_path=exec_path) print("Connected") else: print("unknown error") driver.get(link) driver.maximize_window() sleep(10) mic_off = True cam_off = True src = driver.page_source sleep(2) #Mic and cam off check count = 0 while True: if count == 10: if notifs: send_alert(custom_msg="Couldnt swtich off camera or mic") break if "Turn on microphone" in src and "Turn on camera" in src: break else: count += 1 driver.refresh() sleep(3) src = driver.page_source ask_to_join = False if "Ask to join" in src: ask_to_join = True #join_class try: j = driver.find_element_by_xpath(join_now_path) j.click() if not ask_to_join: if notifs: send_alert(joined_class=True, class_name=class_name) while ask_to_join: sleep( 30 ) #wait 30 seconds for someone to accept (10*30 seconds = 5 minutes) src = driver.page_source if "Ask to join" not in src: ask_to_join = False if notifs: send_alert(joined_class=True, class_name=class_name) if count == 10 or "You can't join this call" in src: if notifs: send_alert(joined_class=False, class_name=class_name) send_alert(custom_msg="Join request was not accepted") except: print("Error\nCouldnt find join button") if notifs: send_alert(joined_class=False, class_name=class_name) #Check for additional join confirmation in case of recording src = driver.page_source if "This meeting is being recorded" in src: try: r = driver.find_element_by_xpath(record_join_now_path) r.click() except: if notifs: send_alert( custom_msg="Joined class but couldnt get past record screen" ) if minimize_after_join: driver.minimize_window() driver.quit() sleep(runtime) #Sleep for the duration of the class #Connect again to exit class try: print("Connecting to driver") driver = webdriver.Chrome(options=chrome_options, executable_path=exec_path) print("Connected") except: print("Couldnt connect to driver") if notifs: send_alert(exit_class=False) #Check if the host ended the meeting before src = driver.page_source if "The meeting has ended" in src: print("Meeting ended by host before end time") if notifs: send_alert(custom_msg="Meeting ended by host before end time") else: try: l = driver.find_element_by_xpath(end_class_path) l.click() if notifs: send_alert(exit_class=True, class_name=class_name) except: print("Error\nCouldnt find the exit button") if notifs: send_alert(exit_class=False, class_name=class_name) if not exit_browser: driver.quit() elif exit_browser: try: driver.close() driver.quit() except: try: process.terminate() except: try: action = ActionChains(driver) action.key_down(Keys.ALT).key_down(Keys.F4).key_up( Keys.F4).key_up(Keys.ALT).perform() except: print("Error\nCouldnt exit the browser") if notifs: send_alert( custom_msg="Error\nCouldnt exit the browser")
from selenium.webdriver.chrome.options import Options options = Options() # 禁止图片和css加载 prefs = { "profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2 } options.add_experimental_option("prefs", prefs) # 无头模式 options.add_argument('--headless')
def newest_post_clicker(): post = driver.find_element_by_css_selector( '#main > div.infinite-scroll > div:nth-child(1) > div.content') title = post.find_element_by_css_selector('h2 > a').text author = post.find_element_by_css_selector( 'p.text-secondary > small:nth-child(4) > a').text click_title = driver.find_element_by_link_text(title) click_title.click() slider = driver.find_element_by_xpath('//*[@id="tipslider"]') email = '*****@*****.**' password = '******' desired_url = 'https://www.publish0x.com/' options = Options() options.add_experimental_option("detach", True) options.add_argument("--window-size=1920,1080") ## options.add_argument("user-data-dir=/Users/vadim/Library/Application Support/BraveSoftware/Brave-Browser") options.binary_location = '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser' driver_path = '/usr/local/bin/chromedriver' driver = webdriver.Chrome(options=options, executable_path=driver_path) driver.get('https://www.publish0x.com/login') waitForLoad('//*[@id="email"]') E_Mail_vak = driver.find_element_by_xpath('//*[@id="email"]') E_Mail_vak.send_keys(email) Pass_vak = driver.find_element_by_xpath('//*[@id="password"]') Pass_vak.send_keys(password) frame = driver.find_element_by_xpath('//iframe[contains(@src, "recaptcha")]') driver.switch_to.frame(frame) Captcha = driver.find_element_by_xpath("//*[@id='recaptcha-anchor']") Captcha.click()
# coding=utf-8 # @Time : 2019/12/22 # @Author: 星空物语 # @File : demo2.py # @Description: 自己测试用,无效文件 from selenium import webdriver from selenium.webdriver.chrome.options import Options # 使用已打开的Chrome浏览器进行调试 # 首先使用命令打开Chrome浏览器并开启远程调试功能 # chrome.exe --remote-debugging-port=9222 --user-data-dir="D:\code\python\selenium_ui_auto\chrome_temp" # 手动打开百度首页,以下代码可以执行成功 chrome_options = Options() chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222") chrome_driver = r"D:\code\python\selenium_ui_auto\driver\chromedriver.exe" driver = webdriver.Chrome(chrome_driver, chrome_options=chrome_options) driver.find_element_by_id('kw').send_keys(u'测试工程师小站')
async def carbon_api(e): """ A Wrapper for carbon.now.sh """ await e.edit("`Processing..`") CARBON = 'https://carbon.now.sh/?l={lang}&code={code}' global CARBONLANG textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcode = str(pcode[8:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded await e.edit("`Processing..\n25%`") if os.path.isfile("./carbon.png"): os.remove("./carbon.png") url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) driver = webdriver.Chrome(executable_path=CHROME_DRIVER, options=chrome_options) driver.get(url) await e.edit("`Processing..\n50%`") download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path } } command_result = driver.execute("send_command", params) driver.find_element_by_xpath("//button[contains(text(),'Export')]").click() driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click() await e.edit("`Processing..\n75%`") # Waiting for downloading while not os.path.isfile("./carbon.png"): await sleep(0.5) await e.edit("`Processing..\n100%`") file = './carbon.png' await e.edit("`Uploading..`") await e.client.send_file( e.chat_id, file, caption="Made using [Carbon](https://carbon.now.sh/about/),\ \na project by [Dawn Labs](https://dawnlabs.io/)", force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove('./carbon.png') driver.quit() # Removing carbon.png after uploading await e.delete() # Deleting msg
from util.settings import Settings from util.datasaver import Datasaver from selenium import webdriver from selenium.webdriver.chrome.options import Options from util.extractor import extract_information, extract_tag_information chrome_options = Options() chrome_options.add_argument("--start-maximized") chrome_options.add_argument("--kiosk") chrome_options.add_argument('--dns-prefetch-disable') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--lang=en-US') chrome_options.add_argument('--headless') chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en-US'}) browser = webdriver.Chrome('./assets/chromedriver', chrome_options=chrome_options) # makes sure slower connections work as well def slackSendMsg(msg): webhook_url = 'https://hooks.slack.com/services/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' slack_data = {'text': '#InstagramCrawl:\n' + msg} response = requests.post( webhook_url, data=json.dumps(slack_data), headers={'Content-Type': 'application/json'} ) if response.status_code != 200: raise ValueError( 'Request to slack returned an error %s, the response is:\n%s'
async def carbon_api(e): """ carbon.now.sh için bir çeşit wrapper """ await e.edit("`İşleniyor...`") CARBON = 'https://carbon.now.sh/?l={lang}&code={code}' global CARBONLANG textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcode = str(pcode[8:]) elif textx: pcode = str(textx.message) # Girilen metin, modüle aktarılıyor. code = quote_plus(pcode) # Çözülmüş url'ye dönüştürülüyor. await e.edit("`İşleniyor...\nTamamlanma Oranı: 25%`") if os.path.isfile("./carbon.png"): os.remove("./carbon.png") url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) driver = webdriver.Chrome(options=chrome_options) driver.get(url) await e.edit("`İşleniyor...\nTamamlanma Oranı: 50%`") download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path } } command_result = driver.execute("send_command", params) driver.find_element_by_xpath("//button[contains(text(),'Export')]").click() # driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() # driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click() await e.edit("`İşleniyor...\nTamamlanma Oranı: 75%`") # İndirme için bekleniyor while not os.path.isfile("./carbon.png"): await sleep(0.5) await e.edit("`İşleniyor...\nTamamlanma Oranı: 100%`") file = './carbon.png' await e.edit("`Resim karşıya yükleniyor...`") await e.client.send_file( e.chat_id, file, caption= "Bu resim [Carbon](https://carbon.now.sh/about/) kullanılarak yapıldı,\ \nbir [Dawn Labs](https://dawnlabs.io/) projesi.", force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove('./carbon.png') driver.quit() # Karşıya yüklemenin ardından carbon.png kaldırılıyor await e.delete() # Mesaj siliniyor
async def carbon_api(e): if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"): """ A Wrapper for carbon.now.sh """ await e.edit("`Processing..`") CARBON = 'https://carbon.now.sh/?l={lang}&code={code}' global CARBONLANG textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcodee = str(pcode[8:]) if "|" in pcodee: pcode, skeme = pcodee.split("|") else: pcode = pcodee skeme = None elif textx: pcode = str(textx.message) skeme = None # Importing message to module code = quote_plus(pcode) # Converting to urlencoded await e.edit("`Making Carbon...\n25%`") url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) driver = webdriver.Chrome(executable_path=CHROME_DRIVER, options=chrome_options) driver.get(url) await e.edit("`Be Patient...\n50%`") download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path } } command_result = driver.execute("send_command", params) driver.find_element_by_xpath( '/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]' ).click() if skeme != None: k_skeme = driver.find_element_by_xpath( '/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input' ) k_skeme.send_keys(skeme) k_skeme.send_keys(Keys.DOWN) k_skeme.send_keys(Keys.ENTER) else: color_scheme = str(random.randint(1, 29)) driver.find_element_by_id( ("downshift-0-item-" + color_scheme)).click() driver.find_element_by_id("export-menu").click() driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() driver.find_element_by_xpath( "//button[contains(text(),'PNG')]").click() await e.edit("`Processing..\n75%`") # Waiting for downloading sleep(2.5) color_name = driver.find_element_by_xpath( '/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input' ).get_attribute('value') await e.edit("`Done...\n100%`") file = './carbon.png' await e.edit("`Uploading..`") await e.client.send_file( e.chat_id, file, caption= "<< `Here's your carbon!` \n**Carbonised by** SudoBot>>\n**Colour Scheme: **`{}`" .format(color_name), force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove('./Carbon.png') driver.quit() # Removing carbon.png after uploading await e.delete() # Deleting msg
def getDatasets(chrome_driver ="/home/twesleyb/src/chromedriver.exe", ''' Collect all available datasets from cells.ucsc.edu. ''' ## Default parameters. LOG_LEVEL = 3 META_EXTENSION = ".tsv" DATA_EXTENSION = ".tsv.gz" BASEURL = "https://cells.ucsc.edu/" CHROME_DRIVER = "/home/twesleyb/src/chromedriver.exe" chrome_driver = CHROME_DRIVER # Chrome options and path to chromedriver. chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument('log-level=' + str(LOG_LEVEL)) chrome_options.add_experimental_option('excludeSwitches',['enable-logging']) # Start headless chromedriver session. driver = webdriver.Chrome(chrome_driver,options=chrome_options) driver.get(BASEURL) xpath = "//a[@role='button'][@class='list-group-item tpDatasetButton']" buttons = driver.find_elements_by_xpath(xpath) data = list() for button in buttons: button.click() sleep(1) tabs = driver.find_elements_by_class_name('tpDatasetTab')
def test_fac(): cwd = os.getcwd() chrome_logpath = os.path.join(cwd, "browser.log") print("CWD: %s log: %s"%(cwd, chrome_logpath)) prefs = { "profile.default_content_settings.popups":0, "download.prompt_for_download": "false", "download.default_directory": cwd, } chrome_options = Options() chrome_options.add_argument("--disable-extensions") #chrome_options.add_argument("--headless") #chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--no-sandbox") chrome_options.add_experimental_option("prefs", prefs) PROXY = "bbpproxy.epfl.ch:80" webdriver.DesiredCapabilities.CHROME['proxy'] = { "httpProxy":PROXY, "ftpProxy":PROXY, "sslProxy":PROXY, "noProxy":None, "proxyType":"MANUAL", "class":"org.openqa.selenium.Proxy", "autodetect":False } if True: driver = webdriver.Chrome(chrome_options=chrome_options, service_args=["--verbose", "--log-path=%s"%chrome_logpath]) else: driver = webdriver.Remote( command_executor='http://bbpsrvc22.bbp.epfl.ch:4444/wd/hub', #desired_capabilities=webdriver.DesiredCapabilities.CHROME) desired_capabilities=chrome_options.to_capabilities()) driver.get("https://www.google.com/ncr") elem = driver.find_element_by_xpath("//input[@id='lst-ib']") elem.send_keys("blue brain project") elem.submit() text = driver.find_element_by_xpath("//div[@id='resultStats']").text print(text) driver.get("https://ufile.io/yo4y6") time.sleep(10) elem = driver.find_element_by_xpath("//a[@class='free-download btn btn-default btn-lg standard-button free-download-button']") driver.execute_script("arguments[0].click();", elem) time.sleep(10) # Open the downloaded zip file import glob import zipfile created_files = glob.glob(os.path.join(cwd, "result.zip")) print(created_files) assert_equal(len(created_files), 1) created_file = created_files[0] z = zipfile.ZipFile(created_file) for f in z.namelist(): print(f)
filepath = os.path.join(dirpath, log) if not os.path.isfile(filepath): with open(filepath, 'w') as f: pass # init logger with open('logging.yaml', 'r') as file: config = yaml.load(file, Loader=yaml.FullLoader) logging.config.dictConfig(config) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # init driver driverpath = os.path.join(os.path.dirname(__file__), 'chromedriver') options = Options() options.add_experimental_option('detach', True) options.add_argument('--ignore-ssl-errors=yes') options.add_argument('--ignore-certificate-errors') driver = webdriver.Chrome(executable_path=driverpath, options=options) def login(): try: driver.get('') driver.find_element(By.NAME, 'id_petugas').send_keys('') driver.find_element(By.NAME, 'password').send_keys('') except (NoSuchElementException) as e: logger.exception(e) driver.quit()
async def carbon_api(e): if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"): """ A Wrapper for carbon.now.sh """ await e.edit("`Processing..`") CARBON = 'https://carbon.now.sh/?l={lang}&code={code}' global CARBONLANG textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcode = str(pcode[8:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded await e.edit("`Meking Carbon...\n25%`") url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) driver = webdriver.Chrome(executable_path=CHROME_DRIVER, options=chrome_options) driver.get(url) await e.edit("`Be Patient...\n50%`") download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path } } command_result = driver.execute("send_command", params) driver.find_element_by_xpath( "//button[contains(text(),'Export')]").click() #driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() #driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click() await e.edit("`Processing..\n75%`") # Waiting for downloading sleep(2.5) await e.edit("`Done Dana Done...\n100%`") file = './carbon.png' await e.edit("`Uploading..`") await e.client.send_file( e.chat_id, file, caption= "<< Here's your carbon, \n Carbonised by [SimbhaUserbot](https://www.github.com/NikhilNGY/SimbhaUserbot)>> ", force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove('./SimbhaUserbot.jpg') driver.quit() # Removing carbon.png after uploading await e.delete() # Deleting msg
async def carbon_api(e): if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"): """ A Wrapper for carbon.now.sh """ await e.edit("Processing...") CARBON = 'https://carbon.now.sh/?l={lang}&code={code}' CARBONLANG = "en" textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcode = str(pcode[8:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = Config.GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument('--disable-gpu') prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) await e.edit("Processing 30%") driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER, options=chrome_options) driver.get(url) download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path } } command_result = driver.execute("send_command", params) driver.find_element_by_xpath( "//button[contains(text(),'Export')]").click() sleep(5) # this might take a bit. driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() sleep(5) await e.edit("Processing 50%") driver.find_element_by_xpath( "//button[contains(text(),'PNG')]").click() sleep(5) #Waiting for downloading await e.edit("Processing 90%") file = './carbon.png' await e.edit("Done!!") await e.client.send_file( e.chat_id, file, caption="《《CARBON by [BorgBot](github.com/ArnabXD/BorgBot)》》", force_document=False, reply_to=e.message.reply_to_msg_id, ) os.remove('./carbon.png') # Removing carbon.png after uploading await e.delete() # Deleting msg
async def carbon_api(e): if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"): """ A Wrapper for carbon.now.sh """ hmm = await e.reply("📛📛📛📛📛") CARBON = 'https://carbon.now.sh/?bg=rgba(239%2C40%2C44%2C1)&t=one-light&wt=none&l=application%2Ftypescript&ds=true&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Hack&fs=14px&lh=143%25&si=false&es=2x&wm=false&code={code}' CARBONLANG = "en" textx = await e.get_reply_message() pcode = e.text if pcode[8:]: pcode = str(pcode[8:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = Config.GOOGLE_CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument('--disable-gpu') prefs = {'download.default_directory': './'} chrome_options.add_experimental_option('prefs', prefs) await hmm.edit("🔘🔘📛📛📛") driver = webdriver.Chrome( executable_path=Config.CHROME_DRIVER, options=chrome_options) driver.get(url) download_path = './' driver.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = {'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': download_path}} driver.execute("send_command", params) driver.find_element_by_xpath( '//*[@id="__next"]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]').click() driver.find_element_by_id("export-menu").click() # driver.find_element_by_xpath("//button[contains(text(),'Export')]").click() sleep(5) # this might take a bit. driver.find_element_by_xpath("//button[contains(text(),'4x')]").click() sleep(5) await hmm.edit("🔘🔘🔘📛📛") driver.find_element_by_xpath( "//button[contains(text(),'PNG')]").click() sleep(5) # Waiting for downloading await hmm.edit("🔘🔘🔘🔘🔘") file = './carbon.png' await hmm.edit("☣️Karbon2 Completed, Uploading Karbon☣️") await e.client.send_file( e.chat_id, file, caption="Karbon2 by [@PhycoNinja13b](https://github.com/Phyco-Ninja/UniNinja)", force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove('./carbon.png') # Removing carbon.png after uploading await hmm.delete() # Deleting msg
async def carbon_api(e): """ A Wrapper for carbon.now.sh """ cat = await edit_or_reply(e, "🌚🌚🌚🌚🌚") CARBON = "https://carbon.now.sh/?bg=rgba(29%2C40%2C104%2C1)&t=one-light&wt=none&l=application%2Ftypescript&ds=true&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Hack&fs=14px&lh=143%25&si=false&es=2x&wm=false&code={code}" CARBONLANG = "en" textx = await e.get_reply_message() pcode = e.text if pcode[6:]: pcode = str(pcode[6:]) elif textx: pcode = str(textx.message) # Importing message to module code = quote_plus(pcode) # Converting to urlencoded url = CARBON.format(code=code, lang=CARBONLANG) chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.binary_location = Config.CHROME_BIN chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-gpu") prefs = {"download.default_directory": "./"} chrome_options.add_experimental_option("prefs", prefs) await cat.edit("🌝🌝🌚🌚🌚") driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER, options=chrome_options) driver.get(url) download_path = "./" driver.command_executor._commands["send_command"] = ( "POST", "/session/$sessionId/chromium/send_command", ) params = { "cmd": "Page.setDownloadBehavior", "params": { "behavior": "allow", "downloadPath": download_path }, } driver.execute("send_command", params) driver.find_element_by_xpath("//button[contains(text(),'Export')]").click() await asyncio.sleep(2) await cat.edit("🌝🌝🌝🌚🌚") # driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click() await asyncio.sleep(2) # Waiting for downloading await cat.edit("🌝🌝🌝🌝🌝") file = "./carbon.png" await cat.edit("✅Karbon4 Completed, Uploading Karbon✅") await e.client.send_file( e.chat_id, file, caption=f"Here's your Karbon4 ", force_document=True, reply_to=e.message.reply_to_msg_id, ) os.remove("./carbon.png") # Removing carbon.png after uploading await cat.delete() # Deleting msg
import os from datetime import datetime from selenium import webdriver from selenium.webdriver.chrome.options import Options from util.cli_helper import get_all_tags from util.extractor import extract_information from util.extractor import get_usernames_from_tag chrome_options = Options() chrome_options.add_argument('--dns-prefetch-disable') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--lang=en-US') # chrome_options.add_argument('--headless') chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en-US'}) browser = webdriver.Chrome('./assets/chromedriver', chrome_options=chrome_options) # makes sure slower connections work as well browser.implicitly_wait(25) def rotate_number(prefix, suffix): i = 1 while os.path.exists(prefix + str(i) + suffix): i += 1 return i def getAllUsers(path):
display = Display(visible=0, size=(800, 800)) display.start() options = Options() options.add_argument("--headless") options.add_argument('--no-sandbox') options.add_argument("--window-size=1920,1080") options.add_argument('--disable-dev-shm-usage') options.add_argument("--disable-extensions") options.add_argument("--disable-gpu") options.add_argument("--enable-javascript") options.add_argument("disable-infobars") options.add_argument("start-maximized") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_argument("--disable-browser-side-navigation") #====================================================================================================# capa = DesiredCapabilities.CHROME capa["pageLoadStrategy"] = "eager" driver = webdriver.Chrome(ChromeDriverManager().install(), options=options, desired_capabilities=capa) print("Headless Chrome Initialized") params = {'behavior': 'allow', 'downloadPath': r''} driver.execute_cdp_cmd('Page.setDownloadBehavior', params) download_dir = '/Downloads'
class DGBRXtractor: def __init__(self): pass def wb_start(self): # Ignorar avisos warnings.filterwarnings('ignore') # Configuração do webdriver self.chrome_options = Options() self.chrome_options.add_argument('--headless') self.chrome_options.add_experimental_option('excludeSwitches', ['enable-logging']) self.browser = webdriver.Chrome( executable_path='./tools/webdrivers/chromedriver_linux.exe', options=self.chrome_options) ''' Este método foi desenvolvido para realizar o download de todos os arquivos do conjunto de dados a partir da sua url. url -> variável do tipo string | Contém a url da página do dataset que possui os arquivos a serem baixados. ext -> variável do tipo lista | Lista de strings com as extensões dos arquivos que deseja-se baixar. Por exemplo: ['.csv', '.xlsx', '.pdf']. O valor padrão é somente ['.csv']. ''' def get_files_by_ds_url(self, url, ext=['.csv']): #Carregar configurações do WebBrowser self.wb_start() # Carregamento da página do dataset print('Carregando [{}]'.format(url)) try: init = time.time() self.browser.get(url) end = time.time() print('Página carregada com sucesso ({} s).'.format(end - init)) except: print( 'Ocorreu um erro ao carregar a página, tente novamente por favor.' ) self.browser.close() sys.exit() # Obter o nome do dataset a partir da url da página dataset_id = self.browser.current_url.split('/')[-1] # Selecionar os elementos HTML <li> que possuem a classe "resource-item" | file_elements é uma lista file_elements = self.browser.find_elements_by_css_selector( 'li.resource-item') # Iterar por todos os elementos armazenados em file_elements for element in file_elements: # Obter o formato do arquivo a ser baixado. Esse elemento se encontra no atributo "data-format" da tag <span> que fica dentro dos <li> que selecionamos anteriormente file_format = '.' + element.find_element_by_css_selector( 'span').get_attribute('data-format') # Transformar arquivos que são denotados como .zip+css ou .zip+xlsx e outros apenas em .zip if file_format.startswith('.zip'): file_format = '.zip' # Verificar se o formato de arquivo raspado está presente na lista de extensões que desejamos baixar if file_format in ext: # Gerar um nome para o arquivo a ser salvo. Aqui foi utilizado o atributo "title" das tags <a> que contém o link para o detalhamento do dataset, que possuem classe "heading" file_name = unidecode( element.find_element_by_css_selector('a.heading'). get_attribute('title').lower().strip().replace( ' ', '-').replace('/', '-')) # Formatar o nome do arquivo para adicionar a extensão if not file_name.endswith(file_format): file_name += file_format # Obter o link para download do arquivo. Ele se encontra no atributo "href" das tags <a> que possuem classe "resource-url-analytics" file_url = element.find_element_by_css_selector( 'a.resource-url-analytics').get_attribute('href') # Verificar se já existe uma pasta no diretório de execução com a identificação do dataset que pegamos lá em cima, caso não exista ela será criada if not os.path.exists(dataset_id): os.makedirs(dataset_id) # Baixar o arquivo utilizando a biblioteca requests print('Downloading {}'.format(file_name)) req = requests.get(file_url) # Definir o local e o nome que o arquivo será salvo file_path = dataset_id + '/' + file_name # Gravar o arquivo with open(file_path, mode='wb') as f: f.write(req.content) f.close() # Fechar o browser self.browser.close() ''' Este método verifica se existe algum tipo de paginação na página atual. ''' def have_pagination(self): try: num_pages = self.browser.find_elements_by_css_selector( 'div.pagination li')[-2].text return num_pages except: return 1 ''' Este método cria um objeto .json contendo todas as organizações presentes no DADOS.GOV.BR, bem como seus URLS. ''' def update_organization_dictionary(self): print('Atualizando dicionário de organizações do DADOS.GOV.BR') #Carregar configurações do WebBrowser self.wb_start() #Carregar a página inicial das organizações url = 'https://dados.gov.br/organization?q=&sort=&page=' try: init = time.time() self.browser.get(url + '1') end = time.time() #print('Página carregada com sucesso ({} s).'.format(end-init)) except: print( 'Ocorreu um erro ao carregar a página para obter elementos de paginação, tente novamente por favor.' ) self.browser.close() sys.exit() #Verificar paginação pages = int(self.have_pagination()) #Variável que irá receber o dicionário organizations = {} #Criar looping por todas as páginas for page in range(1, pages + 1): #Carregar a página try: init = time.time() completed_url = url + str(page) self.browser.get(completed_url) end = time.time() print('Página [{}] carregada com sucesso ({} s).'.format( completed_url, end - init)) except: print( 'Ocorreu um erro ao carregar a página, tente novamente por favor.' ) self.browser.close() sys.exit() #Coletar as informações SIGLA, NOME e URL cards = self.browser.find_elements_by_css_selector( 'li.media-item a') for card in cards: org_name = card.get_attribute('title') org_initials = org_name.split(' ')[-1] org_url = card.get_attribute('href') organizations[org_initials] = { 'name': org_name, 'url': org_url } with open('org_dictionary.json', 'wb') as f: f.write(json.dumps(organizations)) f.close() print('Atualização realizada com êxito.') ''' Este método tem como finalidade recolher todos os links de datasets de uma organização, por meio do link da página desta organização no DADOS.GOV.BR. org_url -> variável do tipo string | URL da página da organização no DADOS.GOV.BR ''' def get_ds_urls_by_organization_url(self, org_url): print('Carregando URLs dos conjuntos de dados...') #Carregar as configurações do WebBrowser self.wb_start() #Carregar a página self.browser.get(org_url) #Verificar paginação pages = int(self.have_pagination()) #Variável que irá armazenar os links datasets = [] for page in range(1, pages + 1): #Carregar a página url = org_url + '?page=' + str(page) #print('carregando... ' + url) self.browser.get(url) #Obter os URLs do dataset ds_url_list = self.browser.find_elements_by_css_selector( 'h3.dataset-heading a') #print(ds_url_list) #Adicionando os links na lista for url in ds_url_list: datasets.append(url.get_attribute('href')) print('Datasets carregados com sucesso.') return datasets
def create_training_data(): """ Get articles and store with share price change over article date. """ # Run/return CLI control run_script = "" while run_script != "y": run_script = input( "\nWARNING: If ..\data\\training_data.csv exists, this script will replace it." " Check that ../data/RR.L.csv is up to date with " "uk.reuters.com/companies/RR.L/news.\nContinue? [y/n]\n") if run_script == "n": print("Returning to main menu...") return elif run_script != "y": print("Invalid input") print( '\n######################\nCreating training data\n######################' ) # Start webdriver and get the article list page print('\nStarting webdriver...') opts = Options() opts.set_headless() opts.add_argument('log-level=3') # Supress log (not fatal errors) opts.add_experimental_option('excludeSwitches', ['enable-logging']) # As above driver = webdriver.Chrome(options=opts) driver.implicitly_wait(120) URL = 'https://uk.reuters.com/companies/RR.L/news' driver.get(URL) # Close cookie pop up print('\nClosing cookie popup...') button = driver.find_element_by_id("_evidon-banner-acceptbutton") button.click() # Scroll down to load required no. of articles and store their anchor # tags print('\nFinding articles...') articles = driver.find_elements_by_class_name( "MarketStoryItem-headline-2cgfz") no_of_articles = len(articles) # Multiples of 20 (there are 20 articles per page) while no_of_articles < 240: driver.find_element_by_tag_name("body").send_keys(Keys.PAGE_DOWN) articles = driver.find_elements_by_class_name( "MarketStoryItem-headline-2cgfz") no_of_articles = len(articles) print("\nNumber of articles found: {}".format(no_of_articles)) # Prep price data csv for reading print('\nGetting share price data...') prices_csv = open('../data/RR.L.csv', 'r', newline='') prices_reader = csv.reader(prices_csv, delimiter=',') next(prices_reader, None) # Skip header # Create list of closing prices and dates price_list = [] for record in prices_reader: if record[4] != 'null': record_list = [datetime.strptime(record[0], '%Y-%m-%d'), record[4]] price_list.append(record_list) prices_csv.close() print('\nNumber of closing price records found: {}'.format( len(price_list))) print('\nCreating training data [memory usage limit 95%]...') # Delete existing training data if it exists try: os.remove('../data/training_data.csv') except OSError: pass # Create training data csv and prepare for writing training_data_csv = open('../data/training_data.csv', 'a', newline='') training_data_writer = csv.writer(training_data_csv) # Loop to extract relevent text from each article data_list = [] for idx, article in enumerate(articles): # Open article link in new tab and switch to it article.send_keys(Keys.CONTROL + Keys.RETURN) driver.switch_to.window(driver.window_handles[1]) # Find and get text from relevent elements, store as string text_list = [] text_list.append( driver.find_element_by_class_name( 'ArticleHeader-headline-NlAqj').text) for para in driver.find_elements_by_class_name( 'ArticleBody-para-TD_9x'): text_list.append(para.text) text = ' '.join(text_list) # Find and get publish date date = driver.find_element_by_class_name( 'ArticleHeader-date-Goy3y').text date = datetime.strptime(date, '%B %d, %Y') # Find index of price record on date of publish, or closest date after price_date_index = np.array([ record[0] - date if record[0] >= date else timedelta.max for record in price_list ]).argmin(0) # Find percentage change from previous close price to next close price perc_change = 100 * ((float(price_list[price_date_index + 1][1]) - float(price_list[price_date_index - 1][1])) / float(price_list[price_date_index - 1][1])) # Assign article classification based on percentage change if abs(perc_change) < 1: classification = 'neutral' elif perc_change < 0: classification = 'decrease' else: classification = 'increase' # Append article record to full data list record = [text, date, perc_change, classification] data_list.append(record) print('\nNumber of articles processed: {}'.format(idx + 1)) # Monitor memory usage to ensure does not exceed set limit of 95% # If memory exceeds 95% save to disk and check again # If continues to exceed 95% close dirver and exit program memory_usage = psutil.virtual_memory().percent print("Current memory usage: {}%".format(memory_usage)) if memory_usage > 95.0: print("Current memory usage exceeds 95%. Saving data to disk..") training_data_writer.writerows(data_list) data_list = [] if memory_usage > 95.0: print("Memory usage still exceeds 95%. Exiting...") driver.close() driver.switch_to.window(driver.window_handles[0]) driver.close() exit() # Close tab and switch back to article list page driver.close() driver.switch_to.window(driver.window_handles[0]) # Once all articles processed close driver driver.close() # Save data list if has not been saved due to exceeding memory limit if len(data_list) > 0: training_data_writer.writerows(data_list) print('\nTraining data created. Number of records in training data: {}'. format(no_of_articles)) training_data_csv.close() print('\nReturning to main menu...') return
from wordcloud import WordCloud, STOPWORDS import module.Database global uid global postcount #NR of posts downloaded per user postcount=200 #facebook dummy account login info facebook_username = "******" facebook_password = "******" #browser settings images disabled chrome_options = Options() chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2}) driver = webdriver.Chrome(chrome_options=chrome_options) reload(sys) sys.setdefaultencoding('utf8') module.Database.createDatabase() conn = sqlite3.connect('facebook.db') conn.text_factory = str def normalize(s): if type(s) == unicode: return s.encode('utf8', 'ignore') else: return str(s) def loginFacebook(driver): driver.implicitly_wait(120)
def get_max_y_value(y_values_list): maximum_y_value = y_values_list[0] for tmp_y_value in y_values_list[1:]: if maximum_y_value < tmp_y_value: maximum_y_value = tmp_y_value return maximum_y_value chrome_options = Options() chrome_options.add_experimental_option('prefs', { 'credentials_enable_service': False, 'profile': { 'password_manager_enabled': False } }) flights_numbers = [] dates = [] driver = webdriver.Chrome(os.getcwd() + '\chromedriver.exe', chrome_options=chrome_options) # First sign-in driver.get('https://www.flightradar24.com') element = driver.find_element_by_css_selector(".premium-menu-title.premium-menu-title-login")
def setUp(self): chromium_path = '/usr/bin/chromium' chromedriver_path = './chromedriver' opts = Options() opts.binary_location = chromium_path opts.add_experimental_option("prefs", { "download.default_directory": "./test_downloaded", "helperApps.neverAsk.saveToDisk": "octet/stream", "directory_upgrade": True, "profile": {"default_content_settings": {"multiple-automatic-downloads": 1}} }) self.driver = webdriver.Chrome(chrome_options=opts, executable_path=chromedriver_path) #self.driver = webdriver.Firefox() url = os.getenv('TEST_URL', config.url) self.driver.get(url) self.__load_pages() # Sets up by redirecting to login page main_page = page.MainPage(self.driver) main_page.click_login_link()
def __init__(self, logger, user_data_dir, timeout): """Initialize chromedriver for use by all other modules. Args: logger: initialized logger object. user_data_dir: string, directory for chrome data. timeout: integer, number of seconds to wait for web pages to load. """ self.timeout = timeout data_dir = os.path.join(os.getcwd(), user_data_dir) self.logger = logger options = Options() data_dir_option = '--user-data-dir=%s' % data_dir options.add_argument(data_dir_option) options.add_argument('--enable-logging') options.add_argument('--v=1') options.add_argument('--lang=en') options.add_argument('--disable-features=UsePasswordSeparatedSigninFlow') options.add_experimental_option("windowTypes", ["webview"]) self.driver = webdriver.Chrome(chrome_options=options) self.action_chain = ActionChains(self.driver) self.window = {} self.window['main'] = self.driver.current_window_handle self.window['logocert'] = None self.ActionPerformed()
def open_browser(self, browser, proxy=None): log_level = DesiredCapabilities.CHROME log_level['loggingPrefs'] = {'browser': 'SEVERE'} chrome_options = Options() chrome_options.add_argument('--no-sandbox') chrome_options.add_experimental_option('prefs', {"profile.managed_default_content_settings.images": 2, "plugins.plugins_disabled": ["Adobe Flash Player"]}) if proxy: chrome_options.add_argument('--proxy-server=%s' % proxy) if browser.lower() == "ie": webdriver.Ie.__init__(self) log("I open IE browser.") elif browser.lower() == "chrome": webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options) log("I open Chrome Browser") elif browser.lower() == "firefox": webdriver.Firefox.__init__(self) log("I open Firefox Browser") elif browser.lower() == 'mobile': mobile_emulation = { "deviceMetrics": {"width": 1920, "height": 980, "pixelRatio": 3.0}, "userAgent": "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 " "(KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53"} chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options) log("I open Mobile Browser")
def configure_driver(): opts = Options() opts.add_argument('--headless') prefs = {"profile.managed_default_content_settings.images": 2} opts.add_experimental_option("prefs", prefs) opts.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36') driver = webdriver.Chrome(chrome_options=opts, executable_path='D:\soft\chromedriver\chromedriver.exe') return driver