def __init__(self, q, profile_dir=None): self.lr = LRequests() self.gsa = GsaCaptcha() self.key_asins = {} self.q = q # options = webdriver.ChromeOptions() options = uc.ChromeOptions() if profile_dir is not None: options.add_argument('--user-data-dir=%s' % profile_dir) prefs = {} # prefs["profile.default_content_settings"] = {"images": 2} # prefs["profile.managed_default_content_settings"] = {"images": 2} # prefs["intl.accept_languages"] = 'en,en_US' # options.add_experimental_option("prefs", prefs) # options.add_argument('--start-maximized') options.add_argument('--blink-settings=imagesEnabled=false') options.add_argument('--lang=en') # self.browser = webdriver.Chrome(options=options) self.browser = uc.Chrome(options=options, driver_executable_path=os.path.join(os.getcwd(), 'chromedriver.exe'), use_subprocess=True) self.wait = WebDriverWait(self.browser, 120)
async def scrap_page( url: HttpUrl, window_size: WindowSize, wait_before_scroll: int, wait_after_scroll: int, ) -> Tuple[PageMetrics, PageType]: chrome_options = uc.ChromeOptions() chrome_options.headless = True chrome_options.add_argument("--headless") chrome_options.add_argument(f"--window-size={window_size}") capbs = DesiredCapabilities.CHROME.copy() capbs["goog:loggingPrefs"] = {"performance": "ALL"} driver = uc.Chrome(options=chrome_options, desired_capabilities=capbs) driver.set_script_timeout(10) driver.get(url) driver.implicitly_wait(wait_before_scroll) try: driver.execute_script( "window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' })" ) except JavascriptException: pass driver.implicitly_wait(wait_after_scroll) page_type = await get_page_type(driver) page_metrics = await get_page_metrics(driver) driver.quit() return page_metrics, page_type
def __init__(self, credentials, chrome_location, logger, debugging=True): self.credentials = credentials self.debugging = debugging self.logger = logger self.logged_in = False options = uc.ChromeOptions() options.binary_location = chrome_location self.driver = uc.Chrome(options=options)
def __init__(self): try: ua = UserAgent() self.user_agent = ua.random except FakeUserAgentError: self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" options = uc.ChromeOptions() # options.add_argument("--user-agent=" + self.user_agent) self.bot = uc.Chrome(options=options) self.bot.delete_all_cookies()
def main(): with Display(visible=False, size=(1366, 768), backend='xvfb') as _display: options = uc.ChromeOptions() options.add_argument('--user-data-dir={}'.format(tempfile.mktemp())) options.add_argument( '--no-first-run --no-service-autorun --password-store=basic') driver = uc.Chrome(version_main=98, options=options) driver.get('http://javabin.cn/bot/bot.html?headless') time.sleep(8) print(driver.find_element_by_tag_name('body').text)
def setup_driver(): """Setup the firefox geckodriver to be headless. Returns: selenium.webdriver.Firefox: The setup geckodriver. """ options = uc.ChromeOptions() # options.headless = True options.add_argument( '--no-first-run --no-service-autorun --password-store=basic') return uc.Chrome(options=options)
async def main(): opts = uc.ChromeOptions() opts.headless = True driver = uc.Chrome(options=opts) streamer = ChromeDriverCV2Streamer(driver) streamer.start() for url in just_some_urls: # with driver: driver.get("https://nu.nl") await asyncio.sleep(3)
def headless_uc(request: FixtureRequest): options = uc.ChromeOptions() options.headless = True request.instance.driver = uc.Chrome(options=options) def teardown(): request.instance.driver.sapipve_screenshot(FAILED_SCREENSHOT_NAME) request.instance.driver.quit() request.addfinalizer(teardown) return request.instance.driver
def __init__(self): # # chop = webdriver.ChromeOptions() # # chop.add_extension(settings.SELENIUM_CHROMEADBLOCK_PATH) # # self.driver = webdriver.Chrome(settings.SELENIUM_CHROMEDRIVER_PATH, chrome_options=chop) # your chosen driver # ffprofile = webdriver.FirefoxProfile() # # ffprofile.add_extension(settings.SELENIUM_GECKOADBLOCK_PATH) # self.driver = webdriver.Firefox(ffprofile, executable_path=settings.SELENIUM_GECKODRIVER_PATH) uc.TARGET_VERSION = 95 options = uc.ChromeOptions() options.add_argument( '--no-first-run --no-service-autorun --password-store=basic') self.driver = uc.Chrome(options=options)
def test_quick(): import undetected_chromedriver.v2 as uc print("uc module: ", uc) # options = selenium.webdriver.ChromeOptions() options = uc.ChromeOptions() options.add_argument("--user-data-dir=c:\\temp") options.binary_location = uc.find_chrome_executable() driver = uc.Chrome( executable_path="./chromedriver.exe", options=options, service_log_path="c:\\temp\\service.log.txt", ) while True: sys.stdin.read()
def open_web(url="https://youtube.com"): global driver try: shutil.rmtree("log") # 쿠키 / 캐쉬파일 삭제 except FileNotFoundError: pass options = uc.ChromeOptions() # setting profile # options.user_data_dir = "log" # another way to set profile is the below (which takes precedence if both variants are used # options.add_argument('--user-data-dir=log') # just some options passing in to skip annoying popups # --password-store=basic --start-maximized options.add_argument('--no-first-run --no-service-autorun') driver = uc.Chrome(options=options) # known url using cloudflare's "under attack mode" driver.get(url) print("[INFO] open_web") time.sleep(3)
def __init__(self, list_queue, product_queue, profile_dir=None): self.list_queue = list_queue self.product_queue = product_queue self.gsa = GsaCaptcha() self.key_asins = {} options = uc.ChromeOptions() if profile_dir: options.add_argument('--user-data-dir=%s' % profile_dir) # options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) # chrome_options = webdriver.ChromeOptions() # prefs = {"profile.managed_default_content_settings.images": 2} prefs = {} prefs["profile.default_content_settings"] = {"images": 2} prefs["profile.managed_default_content_settings"] = {"images": 2} options.add_experimental_option("prefs", prefs) options.add_argument('--start-maximized') options.add_argument('--blink-settings=imagesEnabled=false') self.browser = uc.Chrome(options=options) self.wait = WebDriverWait(self.browser, 120)
def __init__(self, headless=False, store_session=True, proxy=None, user_data=False): self.email = None self.may_send_email = False self.session_data = { "duration": 0, "like": 0, "dislike": 0, "superlike": 0 } start_session = time.time() # this function will run when the session ends @atexit.register def cleanup(): # End session duration seconds = int(time.time() - start_session) self.session_data["duration"] = seconds # add session data into a list of messages lines = [] for key in self.session_data: message = "{}: {}".format(key, self.session_data[key]) lines.append(message) # print out the statistics of the session try: box = self._get_msg_box(lines=lines, title="Tinderbotz") print(box) finally: print("Started session: {}".format(self.started)) y = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) print("Ended session: {}".format(y)) # Go further with the initialisation # Setting some options of the browser here below options = uc.ChromeOptions() # Create empty profile to avoid annoying Mac Popup if store_session: if not user_data: user_data = f"{Path().absolute()}/chrome_profile/" if not os.path.isdir(user_data): os.mkdir(user_data) Path(f'{user_data}First Run').touch() options.add_argument(f"--user-data-dir={user_data}") options.add_argument( '--no-first-run --no-service-autorun --password-store=basic') options.add_argument("--lang=en-GB") if headless: options.headless = True if proxy: if '@' in proxy: parts = proxy.split('@') user = parts[0].split(':')[0] pwd = parts[0].split(':')[1] host = parts[1].split(':')[0] port = parts[1].split(':')[1] extension = get_proxy_extension(PROXY_HOST=host, PROXY_PORT=port, PROXY_USER=user, PROXY_PASS=pwd) options.add_extension(extension) else: options.add_argument(f'--proxy-server=http://{proxy}') # Getting the chromedriver from cache or download it from internet print("Getting ChromeDriver ...") self.browser = uc.Chrome( options=options) #ChromeDriverManager().install(), #self.browser = webdriver.Chrome(options=options) self.browser.set_window_size(1250, 750) # clear the console based on the operating system you're using os.system('cls' if os.name == 'nt' else 'clear') # Cool banner print(Printouts.BANNER.value) time.sleep(1) self.started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) print("Started session: {}\n\n".format(self.started))
def set_viewport_size(driver): width = random.randint(1266, 1833) height = random.randint(856, widthSmallerThen(width, 1080)) window_size = driver.execute_script( """ return [window.outerWidth - window.innerWidth + arguments[0], window.outerHeight - window.innerHeight + arguments[1]]; """, width, height) driver.set_window_size(*window_size) # options = webdriver.ChromeOptions() # options.add_argument("--start-maximized") # options.add_argument(f'user-agent={userAgent}') # driver = webdriver.Chrome(executable_path=PATH, chrome_options=options) options = uc.ChromeOptions() #options.add_argument("--start-maximized") #options.add_argument(f'user-agent={userAgent}') driver = uc.Chrome(options=options) set_viewport_size(driver) driver.delete_all_cookies() def loginCMC(): try: emailInput = WebDriverWait(driver, 5).until( EC.presence_of_element_located( (By.XPATH, '/html/body/div[2]/div/div[2]/div[3]/input[@type="email"]'))) emailInput.clear() emailInput.send_keys(username)
def by_ticker_command(ticker="", sort="fund_percent", num: int = 15): """Display ETF Holdings. [Source: StockAnalysis]""" # Debug if imps.DEBUG: logger.debug("etfs") options = uc.ChromeOptions() options.headless = True options.add_argument("--headless") options.add_argument("--incognito") driver = uc.Chrome(options=options, version_main=98) driver.set_window_size(1920, 1080) driver.get(f"http://etf.com/stock/{ticker.upper()}/") time.sleep(2) driver.find_element(By.XPATH, "(//div[@id='inactiveResult'])[3]").click() soup5 = bs4.BeautifulSoup(driver.page_source, "html.parser") r_ticker, r_holdings, r_name, r_market = [], [], [], [] table1 = soup5.find("table", id="StockTable") table = table1.find("tbody") for x in table.find_all("tr"): r_ticker.append(x.find("td").text) r_name.append(x.find("td").findNext("td").text) r_holdings.append( x.find("td").findNext("td").findNext("td").findNext("td").text) r_market.append( x.find("td").findNext("td").findNext("td").findNext("td").findNext( "td").text) driver.quit() df = pd.DataFrame({ "Ticker": r_ticker, "Name": r_name, "Holdings": r_holdings, "Market Value": r_market, }) if df.empty: raise Exception("No company holdings found!\n") if sort == "mkt_value": key = imps.natsort.index_natsorted( df["Market Value"], key=lambda x: imps.unit_finder.sub(imps.unit_replacer, x), reverse=True, ) df = df.reindex(key) df = df.iloc[:num] df.set_index("Ticker", inplace=True) title = f"ETF's Holding {ticker.upper()}" dindex = len(df.index) if dindex > 15: embeds: list = [] # Output i, i2, end = 0, 0, 15 df_pg, embeds_img, images_list = [], [], [] while i < dindex: df_pg = df.iloc[i:end] df_pg.append(df_pg) fig = imps.plot_df( df_pg, fig_size=(800, (40 * dindex)), col_width=[0.6, 3.5, 0.65, 1.1], tbl_header=imps.PLT_TBL_HEADER, tbl_cells=imps.PLT_TBL_CELLS, font=imps.PLT_TBL_FONT, row_fill_color=imps.PLT_TBL_ROW_COLORS, paper_bgcolor="rgba(0, 0, 0, 0)", ) fig.update_traces(cells=(dict( align=["center", "center", "center", "right"]))) imagefile = "etf-byticker.png" imagefile = imps.save_image(imagefile, fig) if imps.IMAGES_URL or not imps.IMG_HOST_ACTIVE: image_link = imps.multi_image(imagefile) images_list.append(imagefile) else: image_link = imps.multi_image(imagefile) embeds_img.append(f"{image_link}", ) embeds.append(disnake.Embed( title=title, colour=imps.COLOR, ), ) i2 += 1 i += 15 end += 15 # Author/Footer for i in range(0, i2): embeds[i].set_author( name=imps.AUTHOR_NAME, url=imps.AUTHOR_URL, icon_url=imps.AUTHOR_ICON_URL, ) embeds[i].set_footer( text=imps.AUTHOR_NAME, icon_url=imps.AUTHOR_ICON_URL, ) i = 0 for i in range(0, i2): embeds[i].set_image(url=embeds_img[i]) i += 1 embeds[0].set_footer(text=f"Page 1 of {len(embeds)}") choices = [ disnake.SelectOption(label="Home", value="0", emoji="🟢"), ] output = { "view": imps.Menu, "title": title, "embed": embeds, "choices": choices, "embeds_img": embeds_img, "images_list": images_list, } else: fig = imps.plot_df( df, fig_size=(800, (40 * dindex)), col_width=[0.6, 3.5, 0.65, 1.1], tbl_header=imps.PLT_TBL_HEADER, tbl_cells=imps.PLT_TBL_CELLS, font=imps.PLT_TBL_FONT, row_fill_color=imps.PLT_TBL_ROW_COLORS, paper_bgcolor="rgba(0, 0, 0, 0)", ) fig.update_traces(cells=(dict( align=["center", "center", "center", "right"]))) imagefile = imps.save_image("etf-holdings.png", fig) output = { "title": title, "imagefile": imagefile, } return output
def DataPull(path): options = uc.ChromeOptions() # options.headless=True # options.add_argument('--headless') driver = uc.Chrome(options=options) final_directory = path + '/TestData' p = {'download.default_directory': final_directory} final_directory = os.path.join(path, r'TestData') print(f'Made {final_directory} folder.') if not os.path.exists(final_directory): os.makedirs(final_directory) countries = ("fr", "de", "gr", "jp", "us") # countries = ( # "ad","ar","au","at","be", # "bo","br","bg","ca","cl", # "co","cr","cy","cz","dk", # "do","ec","sv","ee","fi", # "fr","de","gr","gt","hn", # "hk","hu","id","is","ie","it", # "jp","lv","li","lt","lu","my", # "mt","mx","mc","nl","nz", # "ni","no","pa","py","pe", # "ph","pl","pt","es","sg", # "sk","se","ch","tw","tr", # "gb","us","uy") # all_markets = { # "AD": "Andorra","AR": "Argentina","AU": "Australia", # "AT": "Austria", # "BE": "Belgium", # "BO": "Bolivia", # "BR": "Brazil", # "BG": "Bulgaria", # "CA": "Canada", # "CL": "Chile", # "CO": "Colombia", # "CR": "Costa Rica", # "CY": "Cyprus", # "CZ": "Czech Republic", # "DK": "Denmark", # "DO": "Dominican Republic", # "EC": "Ecuador", # "SV": "El Salvador", # "EE": "Estonia", # "FI": "Finland", # "FR": "France", # "DE": "Germany", # "GR": "Greece", # "GT": "Guatemala", # "HN": "Honduras", # "HK": "Hong Kong", # "HU": "Hungary", # "ID": "Indonesia", # "IS": "Iceland", # "IE": "Republic of Ireland", # "IT": "Italy", # "JP": "Japan", # "LV": "Latvia", # "LI": "Liechtenstein", # "LT": "Lithuania", # "LU": "Luxembourg", # "MY": "Malaysia", # "MT": "Malta", # "MX": "Mexico", # "MC": "Monaco", # "NL": "Netherlands", # "NZ": "New Zealand", # "NI": "Nicaragua", # "NO": "Norway", # "PA": "Panama", # "PY": "Paraguay", # "PE": "Peru", # "PH": "Philippines", # "PL": "Poland", # "PT": "Portugal", # "ES": "Spain", # "SG": "Singapore", # "SK": "Slovakia", # "SE": "Sweden", # "CH": "Switzerland", # "TW": "Taiwan", # "TR": "Turkey", # "GB": "United Kingdom", # "US": "United States", # "UY": "Uruguay" # }; for country in countries: driver.get('https://spotifycharts.com/regional/{}/daily/latest'.format( country)) try: l = driver.find_element_by_link_text('DOWNLOAD TO CSV') set1 = l set1.click() except NoSuchElementException: continue time.sleep(5) driver.get( 'https://spotifycharts.com/regional/{}/weekly/latest'.format( country)) try: l = driver.find_element_by_link_text('DOWNLOAD TO CSV') set1 = l set1.click() except NoSuchElementException: continue time.sleep(5) timestr = time.strftime("%Y%m%d") downloads_path = str(Path.home() / "Downloads") os.chdir(downloads_path) for file in glob.glob("*daily-latest.csv"): shutil.move(file, final_directory) print(f'Moved {file} Files.') for file in glob.glob("*weekly-latest.csv"): shutil.move(file, final_directory) print(f'Moved {file} Files.') os.chdir(final_directory) os.rename('regional-{}-daily-latest.csv'.format(country), "{}_top200_daily_{}.csv".format(country, timestr)) os.rename('regional-{}-weekly-latest.csv'.format(country), "{}_top_200_weekly_{}.csv".format(country, timestr)) driver.quit()