def __init__(self, q, profile_dir=None):
self.lr = LRequests()
self.gsa = GsaCaptcha()
self.key_asins = {}
self.q = q
# options = webdriver.ChromeOptions()
options = uc.ChromeOptions()
if profile_dir is not None:
options.add_argument('--user-data-dir=%s' % profile_dir)
prefs = {}
# prefs["profile.default_content_settings"] = {"images": 2}
# prefs["profile.managed_default_content_settings"] = {"images": 2}
# prefs["intl.accept_languages"] = 'en,en_US'
# options.add_experimental_option("prefs", prefs)
# options.add_argument('--start-maximized')
options.add_argument('--blink-settings=imagesEnabled=false')
options.add_argument('--lang=en')
# self.browser = webdriver.Chrome(options=options)
self.browser = uc.Chrome(options=options, driver_executable_path=os.path.join(os.getcwd(), 'chromedriver.exe'), use_subprocess=True)
self.wait = WebDriverWait(self.browser, 120)
async def scrap_page(
url: HttpUrl,
window_size: WindowSize,
wait_before_scroll: int,
wait_after_scroll: int,
) -> Tuple[PageMetrics, PageType]:
chrome_options = uc.ChromeOptions()
chrome_options.headless = True
chrome_options.add_argument("--headless")
chrome_options.add_argument(f"--window-size={window_size}")
capbs = DesiredCapabilities.CHROME.copy()
capbs["goog:loggingPrefs"] = {"performance": "ALL"}
driver = uc.Chrome(options=chrome_options, desired_capabilities=capbs)
driver.set_script_timeout(10)
driver.get(url)
driver.implicitly_wait(wait_before_scroll)
try:
driver.execute_script(
"window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' })"
)
except JavascriptException:
pass
driver.implicitly_wait(wait_after_scroll)
page_type = await get_page_type(driver)
page_metrics = await get_page_metrics(driver)
driver.quit()
return page_metrics, page_type
def __init__(self, credentials, chrome_location, logger, debugging=True):
self.credentials = credentials
self.debugging = debugging
self.logger = logger
self.logged_in = False
options = uc.ChromeOptions()
options.binary_location = chrome_location
self.driver = uc.Chrome(options=options)
def __init__(self):
try:
ua = UserAgent()
self.user_agent = ua.random
except FakeUserAgentError:
self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
options = uc.ChromeOptions()
# options.add_argument("--user-agent=" + self.user_agent)
self.bot = uc.Chrome(options=options)
self.bot.delete_all_cookies()
def main():
with Display(visible=False, size=(1366, 768), backend='xvfb') as _display:
options = uc.ChromeOptions()
options.add_argument('--user-data-dir={}'.format(tempfile.mktemp()))
options.add_argument(
'--no-first-run --no-service-autorun --password-store=basic')
driver = uc.Chrome(version_main=98, options=options)
driver.get('http://javabin.cn/bot/bot.html?headless')
time.sleep(8)
print(driver.find_element_by_tag_name('body').text)
def setup_driver():
"""Setup the firefox geckodriver to be headless.
Returns:
selenium.webdriver.Firefox: The setup geckodriver.
"""
options = uc.ChromeOptions()
# options.headless = True
options.add_argument(
'--no-first-run --no-service-autorun --password-store=basic')
return uc.Chrome(options=options)
async def main():
opts = uc.ChromeOptions()
opts.headless = True
driver = uc.Chrome(options=opts)
streamer = ChromeDriverCV2Streamer(driver)
streamer.start()
for url in just_some_urls:
# with driver:
driver.get("https://nu.nl")
await asyncio.sleep(3)
def headless_uc(request: FixtureRequest):
options = uc.ChromeOptions()
options.headless = True
request.instance.driver = uc.Chrome(options=options)
def teardown():
request.instance.driver.sapipve_screenshot(FAILED_SCREENSHOT_NAME)
request.instance.driver.quit()
request.addfinalizer(teardown)
return request.instance.driver
def __init__(self):
# # chop = webdriver.ChromeOptions()
# # chop.add_extension(settings.SELENIUM_CHROMEADBLOCK_PATH)
# # self.driver = webdriver.Chrome(settings.SELENIUM_CHROMEDRIVER_PATH, chrome_options=chop) # your chosen driver
# ffprofile = webdriver.FirefoxProfile()
# # ffprofile.add_extension(settings.SELENIUM_GECKOADBLOCK_PATH)
# self.driver = webdriver.Firefox(ffprofile, executable_path=settings.SELENIUM_GECKODRIVER_PATH)
uc.TARGET_VERSION = 95
options = uc.ChromeOptions()
options.add_argument(
'--no-first-run --no-service-autorun --password-store=basic')
self.driver = uc.Chrome(options=options)
def test_quick():
import undetected_chromedriver.v2 as uc
print("uc module: ", uc)
# options = selenium.webdriver.ChromeOptions()
options = uc.ChromeOptions()
options.add_argument("--user-data-dir=c:\\temp")
options.binary_location = uc.find_chrome_executable()
driver = uc.Chrome(
executable_path="./chromedriver.exe",
options=options,
service_log_path="c:\\temp\\service.log.txt",
)
while True:
sys.stdin.read()
def open_web(url="https://youtube.com"):
global driver
try:
shutil.rmtree("log") # 쿠키 / 캐쉬파일 삭제
except FileNotFoundError:
pass
options = uc.ChromeOptions()
# setting profile
# options.user_data_dir = "log"
# another way to set profile is the below (which takes precedence if both variants are used
# options.add_argument('--user-data-dir=log')
# just some options passing in to skip annoying popups
# --password-store=basic --start-maximized
options.add_argument('--no-first-run --no-service-autorun')
driver = uc.Chrome(options=options)
# known url using cloudflare's "under attack mode"
driver.get(url)
print("[INFO] open_web")
time.sleep(3)
def __init__(self, list_queue, product_queue, profile_dir=None):
self.list_queue = list_queue
self.product_queue = product_queue
self.gsa = GsaCaptcha()
self.key_asins = {}
options = uc.ChromeOptions()
if profile_dir:
options.add_argument('--user-data-dir=%s' % profile_dir)
# options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
# chrome_options = webdriver.ChromeOptions()
# prefs = {"profile.managed_default_content_settings.images": 2}
prefs = {}
prefs["profile.default_content_settings"] = {"images": 2}
prefs["profile.managed_default_content_settings"] = {"images": 2}
options.add_experimental_option("prefs", prefs)
options.add_argument('--start-maximized')
options.add_argument('--blink-settings=imagesEnabled=false')
self.browser = uc.Chrome(options=options)
self.wait = WebDriverWait(self.browser, 120)
def __init__(self,
headless=False,
store_session=True,
proxy=None,
user_data=False):
self.email = None
self.may_send_email = False
self.session_data = {
"duration": 0,
"like": 0,
"dislike": 0,
"superlike": 0
}
start_session = time.time()
# this function will run when the session ends
@atexit.register
def cleanup():
# End session duration
seconds = int(time.time() - start_session)
self.session_data["duration"] = seconds
# add session data into a list of messages
lines = []
for key in self.session_data:
message = "{}: {}".format(key, self.session_data[key])
lines.append(message)
# print out the statistics of the session
try:
box = self._get_msg_box(lines=lines, title="Tinderbotz")
print(box)
finally:
print("Started session: {}".format(self.started))
y = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print("Ended session: {}".format(y))
# Go further with the initialisation
# Setting some options of the browser here below
options = uc.ChromeOptions()
# Create empty profile to avoid annoying Mac Popup
if store_session:
if not user_data:
user_data = f"{Path().absolute()}/chrome_profile/"
if not os.path.isdir(user_data):
os.mkdir(user_data)
Path(f'{user_data}First Run').touch()
options.add_argument(f"--user-data-dir={user_data}")
options.add_argument(
'--no-first-run --no-service-autorun --password-store=basic')
options.add_argument("--lang=en-GB")
if headless:
options.headless = True
if proxy:
if '@' in proxy:
parts = proxy.split('@')
user = parts[0].split(':')[0]
pwd = parts[0].split(':')[1]
host = parts[1].split(':')[0]
port = parts[1].split(':')[1]
extension = get_proxy_extension(PROXY_HOST=host,
PROXY_PORT=port,
PROXY_USER=user,
PROXY_PASS=pwd)
options.add_extension(extension)
else:
options.add_argument(f'--proxy-server=http://{proxy}')
# Getting the chromedriver from cache or download it from internet
print("Getting ChromeDriver ...")
self.browser = uc.Chrome(
options=options) #ChromeDriverManager().install(),
#self.browser = webdriver.Chrome(options=options)
self.browser.set_window_size(1250, 750)
# clear the console based on the operating system you're using
os.system('cls' if os.name == 'nt' else 'clear')
# Cool banner
print(Printouts.BANNER.value)
time.sleep(1)
self.started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print("Started session: {}\n\n".format(self.started))
def set_viewport_size(driver):
width = random.randint(1266, 1833)
height = random.randint(856, widthSmallerThen(width, 1080))
window_size = driver.execute_script(
"""
return [window.outerWidth - window.innerWidth + arguments[0],
window.outerHeight - window.innerHeight + arguments[1]];
""", width, height)
driver.set_window_size(*window_size)
# options = webdriver.ChromeOptions()
# options.add_argument("--start-maximized")
# options.add_argument(f'user-agent={userAgent}')
# driver = webdriver.Chrome(executable_path=PATH, chrome_options=options)
options = uc.ChromeOptions()
#options.add_argument("--start-maximized")
#options.add_argument(f'user-agent={userAgent}')
driver = uc.Chrome(options=options)
set_viewport_size(driver)
driver.delete_all_cookies()
def loginCMC():
try:
emailInput = WebDriverWait(driver, 5).until(
EC.presence_of_element_located(
(By.XPATH,
'/html/body/div[2]/div/div[2]/div[3]/input[@type="email"]')))
emailInput.clear()
emailInput.send_keys(username)
def by_ticker_command(ticker="", sort="fund_percent", num: int = 15):
"""Display ETF Holdings. [Source: StockAnalysis]"""
# Debug
if imps.DEBUG:
logger.debug("etfs")
options = uc.ChromeOptions()
options.headless = True
options.add_argument("--headless")
options.add_argument("--incognito")
driver = uc.Chrome(options=options, version_main=98)
driver.set_window_size(1920, 1080)
driver.get(f"http://etf.com/stock/{ticker.upper()}/")
time.sleep(2)
driver.find_element(By.XPATH, "(//div[@id='inactiveResult'])[3]").click()
soup5 = bs4.BeautifulSoup(driver.page_source, "html.parser")
r_ticker, r_holdings, r_name, r_market = [], [], [], []
table1 = soup5.find("table", id="StockTable")
table = table1.find("tbody")
for x in table.find_all("tr"):
r_ticker.append(x.find("td").text)
r_name.append(x.find("td").findNext("td").text)
r_holdings.append(
x.find("td").findNext("td").findNext("td").findNext("td").text)
r_market.append(
x.find("td").findNext("td").findNext("td").findNext("td").findNext(
"td").text)
driver.quit()
df = pd.DataFrame({
"Ticker": r_ticker,
"Name": r_name,
"Holdings": r_holdings,
"Market Value": r_market,
})
if df.empty:
raise Exception("No company holdings found!\n")
if sort == "mkt_value":
key = imps.natsort.index_natsorted(
df["Market Value"],
key=lambda x: imps.unit_finder.sub(imps.unit_replacer, x),
reverse=True,
)
df = df.reindex(key)
df = df.iloc[:num]
df.set_index("Ticker", inplace=True)
title = f"ETF's Holding {ticker.upper()}"
dindex = len(df.index)
if dindex > 15:
embeds: list = []
# Output
i, i2, end = 0, 0, 15
df_pg, embeds_img, images_list = [], [], []
while i < dindex:
df_pg = df.iloc[i:end]
df_pg.append(df_pg)
fig = imps.plot_df(
df_pg,
fig_size=(800, (40 * dindex)),
col_width=[0.6, 3.5, 0.65, 1.1],
tbl_header=imps.PLT_TBL_HEADER,
tbl_cells=imps.PLT_TBL_CELLS,
font=imps.PLT_TBL_FONT,
row_fill_color=imps.PLT_TBL_ROW_COLORS,
paper_bgcolor="rgba(0, 0, 0, 0)",
)
fig.update_traces(cells=(dict(
align=["center", "center", "center", "right"])))
imagefile = "etf-byticker.png"
imagefile = imps.save_image(imagefile, fig)
if imps.IMAGES_URL or not imps.IMG_HOST_ACTIVE:
image_link = imps.multi_image(imagefile)
images_list.append(imagefile)
else:
image_link = imps.multi_image(imagefile)
embeds_img.append(f"{image_link}", )
embeds.append(disnake.Embed(
title=title,
colour=imps.COLOR,
), )
i2 += 1
i += 15
end += 15
# Author/Footer
for i in range(0, i2):
embeds[i].set_author(
name=imps.AUTHOR_NAME,
url=imps.AUTHOR_URL,
icon_url=imps.AUTHOR_ICON_URL,
)
embeds[i].set_footer(
text=imps.AUTHOR_NAME,
icon_url=imps.AUTHOR_ICON_URL,
)
i = 0
for i in range(0, i2):
embeds[i].set_image(url=embeds_img[i])
i += 1
embeds[0].set_footer(text=f"Page 1 of {len(embeds)}")
choices = [
disnake.SelectOption(label="Home", value="0", emoji="🟢"),
]
output = {
"view": imps.Menu,
"title": title,
"embed": embeds,
"choices": choices,
"embeds_img": embeds_img,
"images_list": images_list,
}
else:
fig = imps.plot_df(
df,
fig_size=(800, (40 * dindex)),
col_width=[0.6, 3.5, 0.65, 1.1],
tbl_header=imps.PLT_TBL_HEADER,
tbl_cells=imps.PLT_TBL_CELLS,
font=imps.PLT_TBL_FONT,
row_fill_color=imps.PLT_TBL_ROW_COLORS,
paper_bgcolor="rgba(0, 0, 0, 0)",
)
fig.update_traces(cells=(dict(
align=["center", "center", "center", "right"])))
imagefile = imps.save_image("etf-holdings.png", fig)
output = {
"title": title,
"imagefile": imagefile,
}
return output
def DataPull(path):
options = uc.ChromeOptions()
# options.headless=True
# options.add_argument('--headless')
driver = uc.Chrome(options=options)
final_directory = path + '/TestData'
p = {'download.default_directory': final_directory}
final_directory = os.path.join(path, r'TestData')
print(f'Made {final_directory} folder.')
if not os.path.exists(final_directory):
os.makedirs(final_directory)
countries = ("fr", "de", "gr", "jp", "us")
# countries = (
# "ad","ar","au","at","be",
# "bo","br","bg","ca","cl",
# "co","cr","cy","cz","dk",
# "do","ec","sv","ee","fi",
# "fr","de","gr","gt","hn",
# "hk","hu","id","is","ie","it",
# "jp","lv","li","lt","lu","my",
# "mt","mx","mc","nl","nz",
# "ni","no","pa","py","pe",
# "ph","pl","pt","es","sg",
# "sk","se","ch","tw","tr",
# "gb","us","uy")
# all_markets = {
# "AD": "Andorra","AR": "Argentina","AU": "Australia",
# "AT": "Austria",
# "BE": "Belgium",
# "BO": "Bolivia",
# "BR": "Brazil",
# "BG": "Bulgaria",
# "CA": "Canada",
# "CL": "Chile",
# "CO": "Colombia",
# "CR": "Costa Rica",
# "CY": "Cyprus",
# "CZ": "Czech Republic",
# "DK": "Denmark",
# "DO": "Dominican Republic",
# "EC": "Ecuador",
# "SV": "El Salvador",
# "EE": "Estonia",
# "FI": "Finland",
# "FR": "France",
# "DE": "Germany",
# "GR": "Greece",
# "GT": "Guatemala",
# "HN": "Honduras",
# "HK": "Hong Kong",
# "HU": "Hungary",
# "ID": "Indonesia",
# "IS": "Iceland",
# "IE": "Republic of Ireland",
# "IT": "Italy",
# "JP": "Japan",
# "LV": "Latvia",
# "LI": "Liechtenstein",
# "LT": "Lithuania",
# "LU": "Luxembourg",
# "MY": "Malaysia",
# "MT": "Malta",
# "MX": "Mexico",
# "MC": "Monaco",
# "NL": "Netherlands",
# "NZ": "New Zealand",
# "NI": "Nicaragua",
# "NO": "Norway",
# "PA": "Panama",
# "PY": "Paraguay",
# "PE": "Peru",
# "PH": "Philippines",
# "PL": "Poland",
# "PT": "Portugal",
# "ES": "Spain",
# "SG": "Singapore",
# "SK": "Slovakia",
# "SE": "Sweden",
# "CH": "Switzerland",
# "TW": "Taiwan",
# "TR": "Turkey",
# "GB": "United Kingdom",
# "US": "United States",
# "UY": "Uruguay"
# };
for country in countries:
driver.get('https://spotifycharts.com/regional/{}/daily/latest'.format(
country))
try:
l = driver.find_element_by_link_text('DOWNLOAD TO CSV')
set1 = l
set1.click()
except NoSuchElementException:
continue
time.sleep(5)
driver.get(
'https://spotifycharts.com/regional/{}/weekly/latest'.format(
country))
try:
l = driver.find_element_by_link_text('DOWNLOAD TO CSV')
set1 = l
set1.click()
except NoSuchElementException:
continue
time.sleep(5)
timestr = time.strftime("%Y%m%d")
downloads_path = str(Path.home() / "Downloads")
os.chdir(downloads_path)
for file in glob.glob("*daily-latest.csv"):
shutil.move(file, final_directory)
print(f'Moved {file} Files.')
for file in glob.glob("*weekly-latest.csv"):
shutil.move(file, final_directory)
print(f'Moved {file} Files.')
os.chdir(final_directory)
os.rename('regional-{}-daily-latest.csv'.format(country),
"{}_top200_daily_{}.csv".format(country, timestr))
os.rename('regional-{}-weekly-latest.csv'.format(country),
"{}_top_200_weekly_{}.csv".format(country, timestr))
driver.quit()
