def driver(pytestconfig):
''' Select and configure a Selenium webdriver for integration tests.
'''
driver_name = pytestconfig.getoption('driver', 'chrome').lower()
if driver_name == "chrome":
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--window-size=1920x1080")
driver = webdriver.Chrome(chrome_options=options)
elif driver_name == "firefox":
from selenium.webdriver.firefox.options import Options
options = Options()
options.add_argument("--headless")
options.add_argument("--window-size=1920x1080")
driver = webdriver.Firefox(firefox_options=options)
elif driver_name == "safari":
driver = webdriver.Safari()
driver.implicitly_wait(10)
yield driver
driver.quit()
def before_all(context):
call( [ "killall", "-9", "chrome" ] )
options = Options()
options.add_extension(path_to_chrome_extension)
#options.add_argument("--remote-debugging-port=9222")
options.add_argument("--start-maximized")
context.driver = webdriver.Chrome(executable_path=path_to_chrome_driver, chrome_options=options)
import platform
import os
import time
import datetime
import random
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
logging.basicConfig(format='[%(levelname)s] [%(asctime)s]: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S %Z',
level=logging.DEBUG)
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--start-maximized')
os_driver = ''
if platform.system() == 'Linux':
os_driver = 'chromedriver81_linux'
elif platform.system() == 'Darwin': # Mac OS
os_driver = 'chromedriver81'
else:
logging.error('''Could not determine which version of Chrome driver
to use due to could not determine OS.''')
logging.info('Found OS as {}'.format(platform.system()))
logging.info(
'Found Chrome driver called {os_driver}'.format(os_driver=os_driver))
def newDriver():
if os.getenv('TRAVIS') == 'true':
options = Options()
options.add_argument("--no-sandbox")
return webdriver.Chrome(chrome_options=options)
return webdriver.Chrome()
def makeConnection(SERVER, USER, PASSWORD):
url = 'https://'+SERVER+'/'
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = '/usr/bin/chromium-browser'
driver = webdriver.Chrome(executable_path= os.path.expanduser('~/usr/local/bin/chromedriver') , chrome_options=chrome_options)
driver.get(url)
time.sleep(1)
driver.save_screenshot(os.path.join(os.path.dirname(os.path.realpath(__file__)), '/tmp', 'kk1.png'))
elemU = driver.find_element_by_name("username")
#while elemU:
print("Identifying...")
elemP = driver.find_element_by_name("password")
elemU.clear()
elemU.send_keys(USER)
elemP.clear()
elemP.send_keys(PASSWORD)
driver.save_screenshot(os.path.join(os.path.dirname(os.path.realpath(__file__)), '/tmp', 'kk2.png'))
elemP.send_keys(Keys.RETURN)
time.sleep(1)
# try:
# elemU = driver.find_element_by_name("username").clear()
# except:
# elemU = None
driver.save_screenshot(os.path.join(os.path.dirname(os.path.realpath(__file__)), '/tmp', 'kk3.png'))
return driver
def setUpClass(self):
#self.driver = webdriver.Ie('c:\\local\\bin\\IEDriverServer.exe')
chrome_options = Options()
chrome_options.add_argument("--start-maximized")
#chrome_options.add_argument("--disable-native-events")
#self.driver = webdriver.Chrome('c:\\local\\bin\\chromedriver.exe',chrome_options=chrome_options)
self.driver = webdriver.Firefox()
def findSnItemPrice(itemId):
itemUrl = urlTemplate.replace('{itemId}', str(itemId))
#print("itemUrl=%s" % itemUrl)
#driver = webdriver.PhantomJS()
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get(itemUrl)
html = driver.page_source
#print(html)
bsObj = BeautifulSoup(html,"html.parser")
priceList=bsObj.findAll("span",{"class":"mainprice"})
itemNameList = bsObj.findAll("h1", {"id":"itemDisplayName"})
itemName = itemNameList[0].get_text()
promDescList = bsObj.findAll("h2", {"id":"promotionDesc"})
promDesc = promDescList[0].get_text()
#print(priceList)
#print(itemNameList)
if len(priceList)>0:
for price in priceList:
strPrice = price.get_text().replace('¥','').replace('.00','')
print("itemId=%s,price:%d,itemName:%s,promDesc:%s" % (itemId,int(strPrice), itemName, promDesc))
else:
strPrice = "-1"
print("itemId=%s,price:%d,itemName:%s,promDesc:%s" % (itemId,int(strPrice), itemName, promDesc))
def __init__(self, user_agent=None, wait_time=2, fullscreen=False,
options=None, **kwargs):
options = Options() if options is None else options
if user_agent is not None:
options.add_argument("--user-agent=" + user_agent)
if fullscreen:
options.add_argument('--kiosk')
prefs = {"download": {
"default_directory": _DOWNLOAD_PATH,
"directory_upgrade": True,
"extensions_to_open": ""
}}
options.add_experimental_option("prefs", prefs)
self.driver = Chrome(chrome_options=options, **kwargs)
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def start_up():
global driver
chrome_options = Options()
chrome_options.add_argument('load-extension=../')
print 'Opening a Chrome browser'
driver = webdriver.Chrome(chrome_options=chrome_options)
time.sleep(1)
def play_video_1(proxy, av):
# print(i)
print(proxy)
# av = "av41724649"
chrome_options = Options()
chrome_options.add_argument("--proxy-server=http://{0}".format(proxy))
drive = webdriver.Chrome(options=chrome_options)
# drive = webdriver.Chrome()
av_url = "https://www.bilibili.com/video/{0}/?spm_id_from=333.788.videocard.4".format(av)
print(av_url)
# drive.get("https://www.bilibili.com/video/av41724649/")
try:
drive.get(av_url)
video = WebDriverWait(drive, 10, 0.5).until(
EC.presence_of_element_located((By.XPATH, "//*[@id='bilibiliPlayer']/div[1]/div[1]/div[8]/video"))) # 找到视频
url = drive.execute_script("return arguments[0].currentSrc;", video) # 打印视频地址
print(url)
print("start")
drive.execute_script("return arguments[0].play()", video) # 开始播放
time.sleep(10)
print("stop")
drive.execute_script("return arguments[0].pause()", video) # 暂停
except Exception as e:
print('except: {0}'.format(e))
finally:
drive.close()
def get_driver(self):
executable_path = self.exec_path
opts = Options()
opts.add_argument('--incognito')
if self.mobile:
opts.add_argument('user-agent=' + _MOBILE_BROWSER_USER_AGENT)
return webdriver.Chrome(self.exec_path, chrome_options = opts)
def instantiate_webdriver(webdriver_class):
if webdriver_class is Firefox:
def webdriver_class():
profile = FirefoxProfile()
# Make sure Firefox WebDriver addon works, even if it could not be verified
profile.set_preference('xpinstall.signatures.required', False)
webdriver = Firefox(profile)
return webdriver
elif webdriver_class is Chrome:
options = ChromeOptions()
# If popup blocking is enabled spawning new window handles via JavaScript won't work!
options.add_argument('disable-popup-blocking')
webdriver_class = functools.partial(webdriver_class, chrome_options=options)
elif webdriver_class is PhantomJS:
def webdriver_class():
webdriver = PhantomJS()
# Fix Selenium PermissionError when trying to delete PhantomJS cookie files by simply not creating a cookie
# file.
webdriver.service._cookie_temp_file = None
return webdriver
try:
return webdriver_class()
# Selenium raises Exception directly in some WebDriver classes...
except Exception:
pytest.skip('WebDriver not available')
def before_feature(context, feature):
chrome_options = Options()
if 'headless' in context.tags:
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1024,768")
context.browser = webdriver.Chrome(
chrome_options=chrome_options,
executable_path="/home/sinan/Downloads/selenium/chromedriver")
# C:\Users\sinan\Downloads\chromedriver_win32_2.0\chromedriver.exe
context.browser.implicitly_wait(3)
""" Kullanacağımız değişkenleri setleyelim. """
context.STOREFRONT = {
"homepage": "https://www.hepsiburada.com",
"cart": "/ayagina-gelsin/sepetim",
"login": "******"
}
context.CATEGORIES = {
"uzaktan_kumanda": "/uzaktan-kumandali-arabalar-c-14001244"
}
context.LOGIN_INFO = {
"mail": "*****@*****.**",
"password": "******"
}
def getKeyFromFile(fname):
if (not os.path.isfile(fname)):
# ok, first we need to do something that will cause us to make this file
chrome_options = Options()
chrome_options.add_argument("--remote-debugging-port=9222")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument("--load-extension=" + unpackedExtensionPath)
chrome_options.add_argument("user-data-dir=" + profilePath) # by insisting on using this profile, we get the fname to exist
driver = webdriver.Chrome(chromeDriverPath, chrome_options=chrome_options, service_args=["--verbose", "--log-path=log.txt"])
driver.close()
f = open(fname, "r")
data = json.load(f)
# print data
if not "extensions" in data:
return None
extensions = data["extensions"]["settings"]
for extension in extensions:
if "path" in extensions[extension]:
path = extensions[extension]["path"]
if "helena" in path:
return extension
return None
def test_home():
#driver = webdriver.Chrome() #for Mac OS
option = Options()
option.add_argument("headless")
driverPath = os.getcwd() + '/chromedriver'
driver = webdriver.Chrome(driverPath, options=option)
driver.get("http://162.246.157.117:8000/")
main_elems = []
elem_names = ["name", "about", "skills", "milk", "education", "butter", "work", "contact"] # testing list
#elem_names = ["name", "about", "skills", "education", "work", "contact"] # actual list
for name in elem_names:
try:
find_elem(name, driver)
main_elems.append((True, name,))
except NoSuchElementException:
main_elems.append((False, name,))
for each in main_elems:
try:
assert each[0] != False
except AssertionError:
print("Missing " + each[1] + "; " + each[1] + " must exist.")
def get_games_id(comp):
dates = [d for d in date_range(START_DATE, END_DATE)]
games_id = []
chrome_options = Options()
chrome_options.add_argument('--dns-prefetch-disable')
driver = Chrome(chrome_options=chrome_options)
for day in dates:
driver.get(
'http://www.espn.com.ar/futbol/resultados/_/liga/{}/fecha/{}'.
format(comp, day)
)
game_link_driver = driver.find_elements_by_class_name(
'mobileScoreboardLink '
)
for game_driver in game_link_driver:
game_id = game_driver.get_attribute('href')[46:53]
games_id.append((game_id, day))
driver.quit
# print(games_id)
return games_id
def getDriver(self, browser, local=False):
if browser == "firefox":
cap = DesiredCapabilities.FIREFOX
if browser == "ie":
cap = DesiredCapabilities.INTERNETEXPLORER
if browser == "chrome":
cap = DesiredCapabilities.CHROME
options = Options()
options.add_argument("--disable-extensions")
options.add_argument("test-type")
if browser == "mobile chrome":
cap = {}
cap['browserName'] = "Chrome"
cap['platformName'] = "Android"
cap['deviceName'] = "android"
if browser == "android":
cap = {}
cap['browserName'] = "Browser"
cap['platformName'] = "Android"
cap['deviceName'] = "android"
if local:
return webdriver.Chrome()
else:
return webdriver.Remote(
command_executor='http://10.238.242.50:4444/wd/hub',
desired_capabilities=cap
)
def __init__(self, options=None, user_agent=None, wait_time=2,
fullscreen=False, incognito=False, headless=False, **kwargs):
options = Options() if options is None else options
if user_agent is not None:
options.add_argument("--user-agent=" + user_agent)
if incognito:
options.add_argument("--incognito")
if fullscreen:
options.add_argument('--kiosk')
if headless:
options.add_argument('--headless')
options.add_argument('--disable-gpu')
self.driver = Chrome(chrome_options=options, **kwargs)
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def _new_webdriver(self, browser=None, *args, **kwargs):
browser = self.expand_browser_name(browser)
# Setup display before creating the browser
self.setup_display()
def append_service_arg(arg):
service_args = kwargs.get('service_args', [])
service_args.append(arg)
kwargs['service_args'] = service_args
if browser == 'PhantomJS':
append_service_arg('--ignore-ssl-errors=true')
if (browser == 'Firefox'
and self.global_settings.get('webdriver_firefox_profile')
and not args and not kwargs.has_key('firefox_profile')):
# Update with profile specified from config
fp = webdriver.FirefoxProfile(self.global_settings.get('webdriver_firefox_profile'))
kwargs['firefox_profile'] = fp
if (browser == 'Chrome' and os.name == 'posix' and os.geteuid() == 0):
self.log.w('Passing --no-sandbox flag to Chrome (running as root)')
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
#self.log.w('Adding --disable-application-cache')
#chrome_options.add_argument('--disable-application-cache')
#chrome_options.add_argument('--incognito')
kwargs['chrome_options'] = chrome_options
driver = getattr(webdriver, browser)(*args, **kwargs)
return driver
def login(username, password):
"""
Using the given username and password, logs into MyNEU through a
selenium driver (firefox). Returns the driver object
"""
opts = Options()
opts.add_argument("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36")
driver = webdriver.Chrome(chrome_options=opts)
# Go to the login page
login_page = driver.get(LOGIN_PAGE)
# Find the user and password forms
user_form = driver.find_element_by_id("username")
pass_form = driver.find_element_by_id("password")
# Login in with the given credentias
user_form.send_keys(username)
pass_form.send_keys(password)
# Click the login button
#button_class = driver.find_element_by_class_name("buttons")
button = driver.find_element_by_class_name("btn-submit")
button.click()
# Also change the size so it all works fine
driver.set_window_size(1280, 800)
# Return the driver so we can do more with it
return driver
def loginToInterface(isMac, chrome, driver, host, port=8686, username='******', password='******'):
logCommon.info('Will start web browser and perform test case.')
chromeDriver = os.path.normpath(driver)
logCommon.info('Browser driver path: ' + str(chromeDriver))
os.environ["webdriver.chrome.driver"] = chromeDriver
opts = Options()
if (not isMac):
opts = Options()
opts.binary_location = os.path.normpath(chrome)
else:
opts.add_argument("--start-maximized")
driver = webdriver.Chrome(chromeDriver, chrome_options=opts)
# options.add_argument("--start-maximized")
# driver.set_window_size(1024, 600)
driver.maximize_window()
# go to the google home page
index = 'http://' + str(host) + ':' + str(port) + '/XOAM/login/index.html'
logCommon.info('Web page: ' + str(index))
driver.get(index)
driver.find_element_by_id('loginUsername').clear()
driver.find_element_by_id('loginUsername').send_keys(username)
driver.find_element_by_id('loginPassword').clear()
driver.find_element_by_id('loginPassword').send_keys(password)
driver.find_element_by_id('submit').click()
try:
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "ebBtnSearch")))
logCommon.info('Login to the InterfaceManagement page successfully.')
except Exception as e:
logCommon.error('Login to the InterfaceManagement page failed.')
return False
return driver
def create_chrome_driver():
chrome_options = Options()
for setting in settings.DEFAULT_REQUEST_HEADERS:
chrome_options.add_argument(''.join((setting, '="', settings.DEFAULT_REQUEST_HEADERS[setting], '"')))
chrome = webdriver.Chrome(executable_path=chrome_path, chrome_options=chrome_options)
chrome.fullscreen_window()
return chrome
def webdriver_options():
options = Options()
options.add_argument('--no-sandbox')
if os.environ.get('SHOW_BROWSER') != '1':
options.add_argument('--headless')
return options
def create_page():
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--start-maximized")
driver = webdriver.Chrome(chrome_options = chrome_options)
driver.get("https://www.mousehuntgame.com")
return driver
def get_chrome_driver(self):
opts = Options()
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def hangout_factory():
if os.environ.get('DISPLAY'):
del os.environ['DISPLAY']
chrome_options = Options()
if 'TRAVIS' in os.environ:
chrome_options.add_argument('--no-sandbox')
hangout = Hangouts(chrome_options=chrome_options)
# hangout.browser.timeout = 60
return hangout
def __init__driver__():
mOptions = Options()
mOptions.add_argument("user-data-dir=./chromeSettings/")
ChromeFacebook.cDriver = webdriver.Chrome(chrome_options=mOptions)
ChromeFacebook.SCREEN_WIDTH = ChromeFacebook.cDriver.execute_script("return screen.width;")
ChromeFacebook.SCREEN_HEIGHT = ChromeFacebook.cDriver.execute_script("return screen.height;")
ChromeFacebook.cDriver.set_window_position(0,0)
ChromeFacebook.cDriver.set_window_size(ChromeFacebook.SCREEN_WIDTH, ChromeFacebook.SCREEN_HEIGHT)
ChromeFacebook.cDriver.get('http://www.facebook.com')
def setUp(self):
chrome_options = Options()
chrome_options.add_argument("--lang=en")
chrome_options.add_argument("--window-size=1277,744")
self.driver = webdriver.Chrome(chrome_options=chrome_options)
self.driver.implicitly_wait(30)
self.base_url = base_url
self.verificationErrors = []
self.accept_next_alert = True
class checker():
def __init__(self,
uid,
passwd,
stdName,
mobile,
form_dir="./form_dir",
code_dir="./code_dir",
os_type="mac",
decoder="baidu",
app_id="",
api_key="",
secret_key="",
maxretry=10):
self.os_type = os_type # mac or linux, not support windows
self.decoder = decoder # "baidu" or "pytesseract"
self.headless = True
""" 你的 APPID AK SK """
self.APP_ID = app_id
self.API_KEY = api_key
self.SECRET_KEY = secret_key
self.header = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
'Connection': 'keep-alive',
'accept': '*/*'
}
self.queryUrl = 'http://stu.sufe.edu.cn/stu/ncp/cn.edu.sufe.ncp.stuReport.queryStdInfo.biz.ext'
self.formUrl = 'http://stu.sufe.edu.cn/stu/ncp/ncpIndex1.jsp'
self.finishedUrl = "http://stu.sufe.edu.cn/stu/ncp/finished.html"
self.submitUrl = 'http://stu.sufe.edu.cn/stu/ncp/cn.edu.sufe.ncp.stuReport.submit.biz.ext'
self.session = requests.session()
self.form_dir = form_dir
self.code_dir = code_dir
self.uid = uid
self.stdName = stdName
self.mobile = mobile
self.passwd = passwd
self.maxretry = maxretry
if self.headless:
from selenium.webdriver.chrome.options import Options # no gui
self.chrome_options = Options()
self.chrome_options.add_argument('--headless')
self.chrome_options.add_argument('--disable-gpu')
self.chrome_options.add_argument('--no-sandbox')
self.chrome_options.add_argument("window-size=1980,1080")
else:
self.chrome_options = None
if self.os_type == "mac":
self.DRIVER_PATH = "./chromeDriver/chromedriver_mac"
elif self.os_type == "linux":
self.DRIVER_PATH = "./chromeDriver/chromedriver_linux"
elif self.os_type == "windows":
self.DRIVER_PATH = "./chromeDriver/chromedriver_win.exe"
else:
print(
'os_type error, please select from ["windows","mac","linux"]')
exit(1)
if self.decoder == "baidu":
self.initial_baidu_aip()
def initial_baidu_aip(self):
from aip import AipOcr
try:
self.client = AipOcr(self.APP_ID, self.API_KEY, self.SECRET_KEY)
if not self.client:
print("百度api没有配置成功，请检查一下")
except Exception as e:
print(e)
exit(1)
def get_valid_code(self, browser):
code_file_name = self.get_code_image(browser)
if self.decoder == "baidu":
code = self.decode_by_baidu(code_file_name)
return code
else:
code = self.decode_by_pytesseract(code_file_name)
return code
def get_login_cookie(self):
retry = 0
while True:
try:
browser = webdriver.Chrome(self.DRIVER_PATH,
options=self.chrome_options)
browser.get(self.queryUrl)
WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.ID, "userId")))
browser.set_window_position(0, 0)
element = browser.find_element_by_class_name("user-login")
element.click()
except Exception as e:
print(e)
exit(1)
valid_code = self.get_valid_code(browser)
if (len(valid_code) < 5):
print("识别到的验证码不符合要求正在重试")
browser.refresh()
continue
try:
browser.find_element_by_id("username").send_keys(self.uid)
browser.find_element_by_id("password").send_keys(self.passwd)
browser.find_element_by_id("imageCodeName").send_keys(
valid_code)
browser.find_element_by_class_name("login").click()
result = browser.find_element_by_tag_name("pre")
# result = WebDriverWait (browser, 5).until (EC.alert_is_present (), "没有消息框，是不是已经填好辽？")
if result:
result = json.loads(result.text)['result'][0]
if result['ISFINISHED'] == 1:
print("报告完成")
exit(0)
else:
break
else:
browser.refresh()
retry += 1
if retry >= self.maxretry:
print("尝试次数达到最大次数，请检查验证码是否解码正确，或用户名密码是否输入正确")
exit(555)
except Exception as e:
browser.refresh()
retry += 1
if retry >= self.maxretry:
print("尝试次数达到最大次数，请检查验证码是否解码正确，或用户名密码是否输入正确")
exit(555)
# 获取cookie
sufeCookies = browser.get_cookies()
browser.quit()
cookies = {}
for item in sufeCookies:
cookies[item['name']] = item['value']
cookiesJar = requests.utils.cookiejar_from_dict(cookies,
cookiejar=None,
overwrite=True)
self.session.cookies = cookiesJar
def get_code_image(self, browser):
try:
_file_name = str(int(time.time()))
_file_name_wz = str(_file_name) + '.png'
_file_url = self.code_dir + '/' + _file_name_wz
browser.get_screenshot_as_file(
_file_url) # get_screenshot_as_file截屏
if not os.path.exists(_file_url):
print("全屏截图失败")
exit(3)
captchaElem = browser.find_element_by_id(
"codeImg").find_element_by_tag_name("img") # # 获取指定元素（验证码）
captchaX = int(captchaElem.location['x'])
captchaY = int(captchaElem.location['y']) # 不知道为什么截屏的验证码需要下移60p
# 获取验证码宽高
captchaWidth = captchaElem.size['width']
captchaHeight = captchaElem.size['height']
captchaRight = captchaX + captchaWidth
captchaBottom = captchaY + captchaHeight
browserWidth = browser.get_window_size()['width']
browserHeight = browser.get_window_size()['height']
tx_p = float(captchaX / browserWidth)
ty_p = float(captchaY / browserHeight)
bx_p = float(captchaRight / browserWidth)
by_p = float(captchaBottom / browserHeight)
imgObject = Image.open(_file_url) # 获得截屏的图片
w, h = imgObject.size
tx = w * tx_p
ty = h * ty_p
bx = w * bx_p
by = h * by_p
imgCaptcha = imgObject.crop((tx, ty, bx, by)) # 裁剪
validcode_file_name = str(_file_name) + 'valid.png'
imgCaptcha.save(self.code_dir + '/' + validcode_file_name)
os.remove(_file_url)
return validcode_file_name
except Exception as e:
print('错误 ：', e)
def checkIfSubmited(self):
try:
flag = self.session.get(
self.queryUrl,
headers=self.header).json()['result'][0]['ISFINISHED'] == 1
return True if flag else False
except Exception as e:
print("检查是否提交时出错", e)
exit(1)
def queryForm(self) -> str:
isFinished = self.checkIfSubmited()
if isFinished:
print("表格填写完成")
exit(0)
else:
time.sleep(1)
response = self.session.get(self.formUrl, headers=self.header).text
soup = BeautifulSoup(response, 'lxml')
if soup.text.find("是否被发现疑似") > -1:
content = "".join(
list(
map(
str,
soup.find_all("div",
attrs={"class":
"weui-cells__title"}))))
digest = hashlib.md5(
content.encode(encoding="gb2312")).hexdigest()
formname = digest + ".json"
return formname
else:
print("登陆错误，请检查cookie!")
exit(1)
def submitForm(self, formName) -> bool:
filePath = self.form_dir + "/" + formName
if os.path.exists(filePath):
with open(filePath, "r") as f:
form = json.loads(f.read())
form["reportDate"] = time.strftime("%Y-%m-%d", time.localtime())
form.update({
"stdCode": self.uid,
"stdName": self.stdName,
"mobile": self.mobile
})
url = self.submitUrl
response = self.session.post(url, headers=self.header, data=form)
if response.status_code == 200:
print("submit 成功")
else:
response.raise_for_status()
else:
print(f"表单信息不存在或可能已经修改，请在{filePath}中手动填写信息")
with open(filePath, 'w') as f:
f.write("请根据stdForm填写该文件")
def _get_file_content(self, filePath):
with open(filePath, 'rb') as fp:
return fp.read()
"""百度aip识别验证码"""
def decode_by_baidu(self, image_name):
""" 读取图片 """
image = self._get_file_content(self.code_dir + "/" + image_name)
""" 调用通用文字识别, 图片参数为本地图片 """
self.client.basicGeneral(image)
""" 带参数调用通用文字识别, 图片参数为本地图片 """
re = self.client.basicGeneral(image)
return re['words_result'][0]['words'].replace(
" ", "") if len(re['words_result']) > 0 else ""
def decode_by_pytesseract(self, code_file_name):
imageCode = Image.open("./code_dir" + "/" + code_file_name) # 图像增强，二值化
imageCode.load()
sharp_img = ImageEnhance.Contrast(imageCode).enhance(2.0)
sharp_img.load()
code = pytesseract.image_to_string(sharp_img).strip().replace(" ", "")
return code
import os
import re
import time
import scrapy
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from CrawlerAffair.utils import process_title, process_time, process_content, process_label
from CrawlerAffair.items import CrawlerAffairItem
# 无头浏览器设置
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument('--no-sandbox')
driver_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'libs',
'chromedriver')
class XinhuaCommonSpider(scrapy.Spider):
"""
xin hua general spider
"""
name = ""
urls = []
allowed_domains = ["news.cn", "xinhuanet.com"]
file_directory=driverDir,
verbose=True,
chmod=True,
overwrite=False,
version=None)
print('Installed chromedriver to path: %s' % path)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
# from selenium.webdriver.common.keys import Keys
options = Options()
options.headless = True # change this to change if browser window opens or not
options.add_argument(
'--log-level=1'
) # used to suppress extraneous messages. Defualt value is 0 (INFO)
# ~~~~~Instantiate Serial Communication~~~~~
import serial
# NOTE the user must ensure that the serial port and baudrate are correct
# serPort = "/dev/ttyS80"
serPort = "COM6" # if having issue with FileNotFound for the port, try increasing the COM port # (e.g. COM6)
baudRate = 9600
ser = serial.Serial(port=serPort, baudrate=baudRate)
print("Serial port " + serPort + " opened Baudrate " + str(baudRate))
startMarker = 60 # '<'
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from utils import *
import epub_writer
# Initialize Colorama
colorama.init(autoreset=True)
# Setup Selenium Webdriver
CHROMEDRIVER_PATH = r"./driver/chromedriver.exe"
options = Options()
options.headless = True
# Disable Warning, Error and Info logs
# Show only fatal errors
options.add_argument("--log-level=3")
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, options=options)
# Get upto which problem it is already scraped from track.conf file
completed_upto = read_tracker("track.conf")
# Load chapters list that stores chapter info
# Store chapter info
with open('chapters.pickle', 'rb') as f:
chapters = pickle.load(f)
def download(problem_num, url, title, solution_slug):
print(Fore.BLACK + Back.CYAN + f"Fetching problem num " + Back.YELLOW +
f" {problem_num} " + Back.CYAN + " with url " + Back.YELLOW +
f" {url} ")
def create_chromedriver(self):
"""
Since we have many functions we can perform we consolidate
chromedriver code here.
"""
# set up options object
chrome_options = Options()
# if we have chrome binary set it up
if self.chrome_binary_path:
chrome_options.binary_location = self.chrome_binary_path
# live dangerously
if self.allow_insecure:
chrome_options.add_argument('--allow-running-insecure-content')
# thank god for this option
chrome_options.add_argument('--mute-audio')
# if we are headless we also mix up window size a bit
if self.headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('disable-gpu')
window_x = random.randrange(1050, 1920)
window_y = random.randrange(900, 1080)
chrome_options.add_argument('window-size=%sx%s' %
(window_x, window_y))
# if we have a ua set it here
if self.ua:
chrome_options.add_argument('user-agent=' + self.ua)
# 'desired_capabilities' is essentially a second way to set options for chrome
# we set loggingPrefs to turn on the performance log which we need to analyze network traffic
# see: https://sites.google.com/a/chromium.org/chromedriver/logging/performance-log
# pageLoadStrategy is set to 'none' to make sure we don't get stuck on pages that never finish loading
# once 'eager' is implemented in chromedriver that may be preferable
# see: https://w3c.github.io/webdriver/#dfn-table-of-page-load-strategies
chrome_capabilities = {
'loggingPrefs': {
'performance': 'ALL'
},
'pageLoadStrategy': 'none'
}
# attempt to start driver, fail gracefull otherwise
try:
# if we have chromedriver path set it up
if self.chromedriver_path:
driver = webdriver.Chrome(
self.chromedriver_path,
desired_capabilities=chrome_capabilities,
chrome_options=chrome_options)
else:
driver = webdriver.Chrome(
desired_capabilities=chrome_capabilities,
chrome_options=chrome_options)
except:
return None
# allow one minute before we kill it, seperate from browser_wait
driver.set_page_load_timeout(self.page_timeout_seconds)
return driver
def StreamPlaylistOrAlbum(self, username, password):
try:
options = Options()
options.add_argument(f'--user-agent={self.GetRandomUserAgent()}')
options.add_argument('--no-sandbox')
options.add_argument('--log-level=3')
options.add_argument('--lang=en')
if self.use_proxy == 1:
options.add_argument('--proxy-server=http://{0}'.format(
self.GetRandomProxyForStream()))
#Removes navigator.webdriver flag
options.add_experimental_option(
'excludeSwitches', ['enable-logging', 'enable-automation'])
# For older ChromeDriver under version 79.0.3945.16
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("window-size=1280,800")
#For ChromeDriver version 79.0.3945.16 or over
options.add_argument(
'--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(options=options)
if self.Login(username, password, driver) == True:
driver.get(self.url)
element_present = EC.presence_of_element_located((
By.XPATH,
'/html/body/div[4]/div/div[2]/div[4]/main/div/div[2]/div/div/div[2]/section/div[4]/div/div[2]/div[2]/div[1]/div/div/div[1]'
))
WebDriverWait(driver, self.max_wait).until(element_present)
index = 0
for i in range(self.number_of_songs):
index += 1
playtime = randint(self.minplay, self.maxplay)
WebDriverWait(driver, self.max_wait).until(
EC.element_to_be_clickable((
By.XPATH,
f'/html/body/div[4]/div/div[2]/div[4]/main/div/div[2]/div/div/div[2]/section/div[4]/div/div[2]/div[2]/div[{index}]/div/div/div[1]'
))).click()
WebDriverWait(driver, self.max_wait).until(
EC.text_to_be_present_in_element((
By.XPATH,
'/html/body/div[4]/div/div[2]/div[3]/footer/div/div[2]/div/div[2]/div[1]'
), '0:01'))
sleep(playtime)
self.PrintText(
Fore.CYAN, Fore.RED, 'PLAYLIST OR ALBUM STREAM',
f'SONG {index} | STREAMED FOR {playtime}s | WITH {username}:{password}'
)
except:
driver.quit()
self.StreamPlaylistOrAlbum(username, password)
finally:
driver.quit()
def get(self, response):
movie_name = self.request.query_params.get('name', None)
type_ = self.request.query_params.get('type', None)
data = find_in_database(movie_name, [])
if movie_name and not data:
options = Options()
options.add_argument("--headless")
browser = webdriver.Chrome(chrome_options=options)
# print(movie_name)
# hotstar = hotstar_search(movie_name, browser, type_)
airtel = airtel_search(movie_name, type_)
eros_now = eros_search(movie_name, type_)
jio = jio_search(movie_name, type_)
idea = idea_search(movie_name, browser)
vodafone = voda_search(movie_name, type_)
zee5 = zee5_search(movie_name, type_)
mx_player = mx_player_search(movie_name, type_)
alt_balaji = alt_balaji_search(movie_name, type_)
# if hotstar:
# for mn, link, typ in hotstar:
# temp = Data(name=mn, provider="hotstar", link=link, movie=typ)
# temp.save()
# data.append(temp)
if airtel:
for mn, link, typ in airtel:
temp = Data(name=mn,
provider="airtel",
link=link,
movie=typ)
temp.save()
if eros_now:
for mn, link, typ in eros_now:
temp = Data(name=mn,
provider="eros_now",
link=link,
movie=typ)
temp.save()
if jio:
for mn, link, typ in jio:
temp = Data(name=mn,
provider="jio_cinema",
link=link,
movie=typ)
temp.save()
if idea:
for mn, link, typ in idea:
temp = Data(name=mn, provider="idea", link=link, movie=typ)
temp.save()
if vodafone:
for mn, link, typ in vodafone:
temp = Data(name=mn,
provider="vodafone",
link=link,
movie=typ)
temp.save()
if zee5:
for mn, link, typ in zee5:
temp = Data(name=mn, provider="zee5", link=link, movie=typ)
temp.save()
if mx_player:
for mn, link, typ in mx_player:
temp = Data(name=mn,
provider="mx_player",
link=link,
movie=typ)
temp.save()
if alt_balaji:
for mn, link, typ in alt_balaji:
temp = Data(name=mn,
provider="alt_balaji",
link=link,
movie=typ)
temp.save()
data = find_in_database(movie_name, [])
if not data:
temp = Data(name=movie_name,
provider=None,
link=None,
movie=False)
temp.save()
serializer = DataSerializer(data, many=True)
return Response(serializer.data)
u"\U0001f926-\U0001f937"
u"\U00010000-\U0010ffff"
u"\u2640-\u2642"
u"\u2600-\u2B55"
u"\u200d"
u"\u23cf"
u"\u23e9"
u"\u231a"
u"\ufe0f" # dingbats
u"\u3030"
"]+",
re.UNICODE)
options = Options()
options.headless = True
options.add_argument("--window-size=1920,1200")
options.add_argument('--no-sandbox')
def getPageText(url):
driver = webdriver.Chrome(options=options)
driver.set_page_load_timeout(30)
driver.get(url)
try:
# Buscar elementos de la clase "pledge_amount"
find_pledges = driver.find_elements_by_css_selector(
'h2.pledge__amount')
pledge_amounts = []
for pledge in find_pledges:
# Para eliminar elementos que no sean los dígitos del monto de contribución
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd
import os
df = pd.read_csv('topic_subtopics.csv')
# set driver options
opt = Options()
opt.add_argument("--incognito")
try:
driver = webdriver.Chrome(executable_path='chromedriver',
chrome_options=opt)
# driver.get("https://www.youtube.com")
# if file does not exist write header
if not os.path.isfile('youtube_links.csv'):
i = 0
j = -1
print("file resources does not exists")
else: # else it exists so append without writing the header, start from the last subtopic covered
i = 0
temp = pd.read_csv('youtube_links.csv')
if (len(temp) > 0):
j = temp["index"].iloc[len(temp) - 1]
else:
j = 0
print("len of df: ", j, " i: ", i)
async def capture(url):
""" For .ss command, capture a website's screenshot and send the photo. """
await url.edit("`Processing ...`")
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--test-type")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(executable_path=CHROME_DRIVER,
options=chrome_options)
input_str = url.pattern_match.group(1)
link_match = match(r'\bhttps?://.*\.\S+', input_str)
if link_match:
link = link_match.group()
else:
await url.edit("`I need a valid link to take screenshots from.`")
return
driver.get(link)
height = driver.execute_script(
"return Math.max(document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight);"
)
width = driver.execute_script(
"return Math.max(document.body.scrollWidth, document.body.offsetWidth, document.documentElement.clientWidth, document.documentElement.scrollWidth, document.documentElement.offsetWidth);"
)
driver.set_window_size(width + 125, height + 125)
wait_for = height / 1000
await url.edit(f"`Generating screenshot of the page...`\
\n`Height of page = {height}px`\
\n`Width of page = {width}px`\
\n`Waiting ({int(wait_for)}s) for the page to load.`")
await sleep(int(wait_for))
im_png = driver.get_screenshot_as_png()
# saves screenshot of entire page
driver.close()
message_id = url.message.id
if url.reply_to_msg_id:
message_id = url.reply_to_msg_id
with io.BytesIO(im_png) as out_file:
out_file.name = "screencapture.png"
await url.edit("`Uploading screenshot as file..`")
await url.client.send_file(url.chat_id,
out_file,
caption=input_str,
force_document=True,
reply_to=message_id)
async def carbon_api(e):
""" A Wrapper for carbon.now.sh """
hell = await edit_or_reply(e, "🌚🌚🌚🌚🌚")
CARBON = "https://carbon.now.sh/?bg=rgba(29%2C40%2C104%2C1)&t=one-light&wt=none&l=application%2Ftypescript&ds=true&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Hack&fs=14px&lh=143%25&si=false&es=2x&wm=false&code={code}"
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[6:]:
pcode = str(pcode[6:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
await hell.edit("🌝🌝🌚🌚🌚")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
await asyncio.sleep(2)
await hell.edit("🌝🌝🌝🌚🌚")
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await asyncio.sleep(2) # Waiting for downloading
await hell.edit("🌝🌝🌝🌝🌝")
file = "./carbon.png"
await hell.edit("✅Karbon4 Completed, Uploading Karbon✅")
await e.client.send_file(
e.chat_id,
file,
caption=f"Here's your Karbon4 ",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
# Removing carbon.png after uploading
await hell.delete() # Deleting msg
async def carbon_api(e):
RED = random.randint(0, 256)
GREEN = random.randint(0, 256)
BLUE = random.randint(0, 256)
THEME = [
"3024-night",
"a11y-dark",
"blackboard",
"base16-dark",
"base16-light",
"cobalt",
"dracula",
"duotone-dark",
"hopscotch",
"lucario",
"material",
"monokai",
"night-owl",
"nord",
"oceanic-next",
"one-light",
"one-dark",
"panda-syntax",
"paraiso-dark",
"seti",
"shades-of-purple",
"solarized",
"solarized%20light",
"synthwave-84",
"twilight",
"verminal",
"vscode",
"yeti",
"zenburn",
]
CUNTHE = random.randint(0, len(THEME) - 1)
The = THEME[CUNTHE]
hell = await edit_or_reply(e, "⬜⬜⬜⬜⬜")
CARBON = "https://carbon.now.sh/?bg=rgba({R}%2C{G}%2C{B}%2C1)&t={T}&wt=none&l=auto&ds=false&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Fira%20Code&fs=14px&lh=152%25&si=false&es=2x&wm=false&code={code}"
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[7:]:
pcode = str(pcode[7:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code,
R=RED,
G=GREEN,
B=BLUE,
T=The,
lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
await hell.edit("⬛⬛⬜⬜⬜")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
await asyncio.sleep(2) # this might take a bit.
# driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
# await asyncio.sleep(5)
await hell.edit("⬛⬛⬛⬜⬜")
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await asyncio.sleep(2) # Waiting for downloading
await hell.edit("⬛⬛⬛⬛⬛")
file = "./carbon.png"
await hell.edit("✅RGB Karbon Completed, Uploading Karbon✅")
await e.client.send_file(
e.chat_id,
file,
caption=f"Here's your karbonrgb",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
await hell.delete() # Deleting msg
def init_browser():
opts = Options()
opts.add_argument("user-data-dir=selenium")
return Chrome(chrome_options=opts)
class inplay():
def __init__(self):
self.Date = datetime.datetime.today().strftime('%Y%m%d')
self.caps = DesiredCapabilities.CHROME
self.caps['loggingPrefs'] = {'performance': 'INFO'}
self.chrome_options = Options()
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36'
self.chrome_options.add_argument(f'user-agent={user_agent}')
# self.chrome_options.add_argument("--headless")
self.driver_path = 'E:/code/chromedriver'
self._INPLAY_BASKETBALL_MS = re.compile(
r'\|EV;AU=0;C1=1;C2=\d{6,8}.*?(?=\|EV;AU=0;)')
self.NA = re.compile(r'(?<=NA=).*?(?=;)') # name
self.CC = re.compile(r'(?<=CC=).*?(?=;)')
self.C2 = re.compile(r'(?<=C2=).*?(?=;)')
self.ID = re.compile(r'(?<=ID=)\w*?(?=;)')
self.r = redis.Redis(host='127.0.0.1', port=6379)
self.driver = webdriver.Chrome(desired_capabilities=self.caps,
executable_path=self.driver_path,
chrome_options=self.chrome_options)
self.driver.set_page_load_timeout(30)
# self.driver.set_window_position(-1500, 1050)
self.wait_60 = WebDriverWait(self.driver, 60)
self.wait_10 = WebDriverWait(self.driver, 10)
self.wait_5 = WebDriverWait(self.driver, 5)
def web(self):
while True:
try:
self.driver.get('https://www.635288.com/')
break
except:
pass
while True:
try:
self.wait_10.until(
EC.element_to_be_clickable(
(By.XPATH, '//a[text()="English"]'))).click()
self.wait_10.until(
EC.element_to_be_clickable(
(By.XPATH, '//div[@title="Live In-Play"]'))).click()
break
except:
self.driver.refresh()
print('重新加载页面')
def overview(self):
while True:
try:
self.wait_60.until(
EC.element_to_be_clickable(
(By.XPATH, '//a[text()="In-Play"]'))).click()
self.wait_60.until(
EC.element_to_be_clickable(
(By.XPATH, '//div[text()="Overview"]'))).click()
break
except:
self.driver.refresh()
print('重新加载页面')
def game_update(self):
while True:
self.driver.refresh()
while True:
flag = False
messages = self.driver.get_log('performance')
for message in messages:
try:
message = json.loads(
message['message']
)['message']['params']['response']['payloadData']
except KeyError:
continue
if '\x14OVInPlay' in message[:10]:
games = self.parse_OVInPlay(message)
past = self.r.smembers('messages')
past_str = set([x.decode() for x in past])
for value in past:
self.r.srem('messages', value)
now = set()
for cell in games:
self.r.sadd('messages', cell['messages'])
self.r.sadd('games', cell['games'])
now.add(cell['messages'])
update = now - past_str
for cell in update:
self.r.sadd('update', cell)
for cell in self.r.smembers('update'):
self.r.srem('update', cell)
flag = True
break
if flag:
break
print('更新比赛嘻嘻！！！')
self.driver.quit()
time.sleep(random.randint(300, 400))
def parse_OVInPlay(self, message):
games = []
basketball = self._INPLAY_BASKETBALL_MS.findall(message)
for cell in basketball:
dict = {}
dict['messages'] = '15' + self.C2.findall(cell)[0] + '5M18_1_3' + '@' + self.ID.findall(cell)[0] + '@' + \
self.NA.findall(cell)[0]
dict['games'] = self.NA.findall(cell)[0]
games.append(dict)
return games
def login(self):
while True:
try:
self.wait_60.until(
EC.element_to_be_clickable(
(By.XPATH,
'//input[@class = "hm-Login_InputField "]')))
break
except:
continue
elements = self.driver.find_elements_by_xpath(
'//input[@class = "hm-Login_InputField "]')
account = elements[0]
pw = elements[1]
account.click()
account.clear()
account.send_keys('pxpwoa')
while True:
try:
pw = self.driver.find_elements_by_xpath(
'//input[@class = "hm-Login_InputField "]')[1]
pw.send_keys('nimabi')
break
except:
pw = self.driver.find_elements_by_xpath(
'//input[@class = "hm-Login_InputField "]')[1]
pw.click()
pw = self.driver.find_elements_by_xpath(
'//input[@class = "hm-Login_InputField "]')[1]
pw.clear()
pw = self.driver.find_elements_by_xpath(
'//input[@class = "hm-Login_InputField "]')[1]
pw.send_keys('nimabi3927493')
pw = self.driver.find_element_by_xpath(
'//button[@class = "hm-Login_LoginBtn "]')
pw.click()
time.sleep(4)
self.close_message()
pass
def login_cookies(self):
self.driver.get('https://www.635288.com/')
self.load_cookies()
self.driver.refresh()
while True:
try:
self.wait_10.until(
EC.element_to_be_clickable((
By.XPATH,
'//div[@class="wl-PushTargetedMessageOverlay_CloseButton "]'
))).click()
self.wait_60.until(
EC.element_to_be_clickable(
(By.XPATH, '//a[text()="In-Play"]'))).click()
self.wait_60.until(
EC.element_to_be_clickable(
(By.XPATH, '//div[text()="Overview"]'))).click()
break
except:
self.driver.refresh()
print('重新加载页面')
def close_message(self):
try:
self.wait_10.until(
EC.element_to_be_clickable((
By.XPATH,
'//div[@class="wl-PushTargetedMessageOverlay_CloseButton "]'
))).click()
except:
pass
def bank(self):
element = self.wait_60.until(
EC.visibility_of_element_located(
(By.XPATH,
'//div[@class = "hm-MembersInfoButton_BankInfo "]')))
rmb = element.text.split(' ')[0]
return rmb
def basketball(self):
self.overview()
element = self.wait_10.until(
EC.element_to_be_clickable(
(By.XPATH, '//div[text()="Basketball"]')))
element.click()
def my_bets(self):
while True:
try:
self.wait_60.until(
EC.visibility_of_element_located(
(By.XPATH, '//div[@class = "bw-BetslipHeader_Item "]'
))).click() ## mybets
self.wait_60.until(
EC.visibility_of_element_located(
(By.XPATH,
'//div[@class = "mbr-OpenBetItemsContainerRhs "]')))
break
except:
self.driver.refresh()
while True:
try:
self.wait_5.until(
EC.element_to_be_clickable(
(By.XPATH,
'//div[@class = "mbr-ShowMoreBetsButtonRhs_Label "]'
))).click()
except:
break
bets_cashOut = self.driver.find_elements_by_xpath(
'//div[@class = "mbr-OpenBetItemRhs "]')
self.driver.find_elements_by_xpath('//div[text()="Live"]')[0].click()
while True:
try:
self.wait_5.until(
EC.element_to_be_clickable(
(By.XPATH,
'//div[@class = "mbr-ShowMoreBetsButtonRhs_Label "]'
))).click()
except:
break
bets_live = self.driver.find_elements_by_xpath(
'//div[@class = "mbr-OpenBetItemRhs "]')
def save_cookies(self):
self.login()
with open('cookies.pickle', 'wb') as filehandler:
pickle.dump(self.driver.get_cookies(), filehandler)
def load_cookies(self):
with open('cookies.pickle', 'rb') as cookiesfile:
cookies = pickle.load(cookiesfile)
for cookie in cookies:
self.driver.add_cookie(cookie)
def set_bets(self, team_home, uint, bet):
team_home = 'Legia Warsaw'
bet = 'U'
element = self.wait_5.until(
EC.visibility_of_element_located(
(By.XPATH, "//div/span[text()='" + team_home + "']")))
parent_elem = element.find_element_by_xpath('..')
parent_elem = parent_elem.find_element_by_xpath('..')
parent_elem = parent_elem.find_element_by_xpath('..')
parent_elem = parent_elem.find_element_by_xpath('..')
parent_elem = parent_elem.find_element_by_xpath('..')
bets = parent_elem.find_element_by_xpath(".//span[contains(text(), '" +
bet + "')]").click()
element.click()
def quick_bet(self):
self.overview()
self.wait_10.until(
EC.visibility_of_element_located((
By.XPATH,
'//div[@class = "bw-BetslipHeader "]/div[text()="Bet Slip"]')))
self.driver.switch_to.frame("bsFrame")
self.wait_60.until(
EC.visibility_of_element_located(
(By.XPATH, "//span[@class='qb-Btn_Switch-false']"))).click()
self.driver.switch_to.default_content()
def refresh(self):
self.driver.refresh()
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
import time
import SendToGoogleSheets
import config
import progressbar
option = Options()
option.add_argument("--disable-infobars")
option.add_argument("start-maximized")
option.add_argument("--disable-extensions")
# Pass the argument 1 to allow and 2 to block
option.add_experimental_option(
"prefs", {"profile.default_content_setting_values.notifications": 1})
driver = webdriver.Chrome(chrome_options=option,
executable_path='chromedriver.exe')
driver.get("https://www.facebook.com/login/")
keyword = ''
def FbLogin():
print('Facebook Login')
username = config.username
password = config.password
async def carbon_api(e):
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
await e.edit("`Processing..`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcodee = str(pcode[8:])
if "|" in pcodee:
pcode, skeme = pcodee.split("|")
else:
pcode = pcodee
skeme = None
elif textx:
pcode = str(textx.message)
skeme = None # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
await e.edit("`Meking Carbon...\n25%`")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=CHROME_DRIVER,
options=chrome_options)
driver.get(url)
await e.edit("`Be Patient...\n50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath(
'/html/body/div[1]/main/div[2]/div[2]/div[1]/div[1]/div/span[2]'
).click()
if skeme != None:
k_skeme = driver.find_element_by_xpath(
'/html/body/div[1]/main/div[2]/div[2]/div[1]/div[1]/div/span[2]/input'
)
k_skeme.send_keys(skeme)
k_skeme.send_keys(Keys.DOWN)
k_skeme.send_keys(Keys.ENTER)
else:
color_scheme = str(random.randint(1, 29))
driver.find_element_by_id(
("downshift-0-item-" + color_scheme)).click()
driver.find_element_by_id("export-menu").click()
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
driver.find_element_by_xpath(
"//button[contains(text(),'PNG')]").click()
await e.edit("`Processing..\n75%`")
# Waiting for downloading
sleep(2.5)
color_name = driver.find_element_by_xpath(
'/html/body/div[1]/main/div[2]/div[2]/div[1]/div[1]/div/span[2]/input'
).get_attribute('value')
await e.edit("`Done Dana Done...\n100%`")
file = './carbon.png'
await e.edit("`Uploading..`")
await e.client.send_file(
e.chat_id,
file,
caption=
"<< `Here's your carbon!` \n **Carbonised by** [@TeleBotHelp.](https://github.com/xditya/TeleBot)>>\n**Colour Scheme: **`{}`"
.format(color_name),
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./Anubis.png')
driver.quit()
# Removing carbon.png after uploading
await e.delete() # Deleting msg
def set_ops():
opts = Options()
ua = generate_user_agent()
opts.add_argument("user-agent=%s" % ua)
return opts
excel_file = None
for file in os.listdir('.'):
if fnmatch.fnmatch(file, '*.xlsx'):
excel_file = file
print('Opening file: ' + '\'' + file.title() + '\'')
if excel_file is not None:
wb = openpyxl.load_workbook(excel_file)
else:
exit('No excel file found')
# First sheet
ws = wb[wb.sheetnames[0]]
# Selenium Chrome Webdriver
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('windows-size=1200x600')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome('chromedriver', options=chrome_options)
# Starting parameters
column_with_urls = input('Enter the letter of the column with urls: ')
column_emails = input('Enter the letter of the column to write emails to: ')
starting_row = int(input('Enter the starting row: '))
number_of_rows = int(input('Enter the number of rows to process: '))
keywords_to_avoid = ['@example', 'example@', 'broofa', '@sentry', 'yourcompany@', 'godaddy', 'placeholder', 'name@',
'@domain', '.png', '.jpg', '.gif', '.jpeg']
return text
def save_img(image_url):
image_name = "{}".format(image_url[-9:])
with open(save_dir + image_name, "wb") as f:
content = requests.get(image_url, headers=headers).content
f.write(content)
# headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"}
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
url = "https://book.douban.com/tag/"
with open("booklist.md", "a") as f:
f.write("# 豆瓣书单 \n\n")
for c in classes:
book_num = 1
with open("booklist.md", "a") as f:
f.write("## {} \n\n".format(c))
kw = urllib.parse.quote(c)
from selenium.webdriver.chrome.options import Options
import time
from difflib import SequenceMatcher
import smtplib
import mimetypes
from email.mime.multipart import MIMEMultipart
from email import encoders
from email.message import Message
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.text import MIMEText
import datetime
chrome_options = Options()
chrome_options.add_argument("--window-size = 1920,1080")
chromeDriver_path = "C:/dev/chromedrivers/chromedriver.exe"
baseUrl_uk="https://www.autotrader.co.uk/car-search?sort=relevance&radius=1500&postcode=e148hl&onesearchad=Used&onesearchad=Nearly%20New&onesearchad=New&keywords=left%20hand%20drive%20lhd&page="
miles_to_km = 1.60934
epsilon_km = 50000
epsilon_years = 2
maxPrice_uk = 30000.0
minDiffPrice = 3000
Nb_pages = 3 #10
max_km = 150000
path = "C:/dev/carScraping/data/"
autotrader = pd.DataFrame(columns = ["title_uk", "brand_uk", "model_uk", "price_uk (gbp)", "year_uk", "kilometrage_uk", "fuelType_uk", "bodyType_uk", "gearBox_uk", "link_uk", "keep_uk"])
leparking = pd.DataFrame(columns = ["brand_fr", "model_fr", "price_fr (eur)", "year_fr", "kilometrage_fr", "fuelType_fr", "gearBox_fr", "link_fr"])
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
chrome_options = Options()
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--use-fake-ui-for-media-stream")
chrome_options.add_experimental_option(
"prefs", {"profile.default_content_setting_values.notifications": 2})
driver = webdriver.Chrome(\
executable_path=r'.\chromedriver_win32\chromedriver.exe',\
options=chrome_options)
driver.get("https://www.sonyliv.com/")
got_video_list = False
while not got_video_list:
current_height = driver.execute_script(
'return document.documentElement.offsetHeight+document.documentElement.scrollTop'
)
scroll_height = driver.execute_script('return document.body.scrollHeight')
if current_height == scroll_height:
async def carbon_api(e):
hell = await edit_or_reply(e, "`Processing....`")
CARBON = "https://carbon.now.sh/?l={lang}&code={code}"
textx = await e.get_reply_message()
pcode = e.text
if pcode[5:]:
pcodee = str(pcode[5:])
if "|" in pcodee:
pcode, skeme = pcodee.split("|")
else:
pcode = pcodee
skeme = None
elif textx:
pcode = str(textx.message)
skeme = None # Importing message to module
pcode = deEmojify(pcode)
code = quote_plus(pcode) # Converting to urlencoded
await hell.edit("`Meking Carbon...`\n`25%`")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
await hell.edit("`Be Patient...\n50%`")
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath(
"/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]"
).click()
if skeme is not None:
k_skeme = driver.find_element_by_xpath(
"/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input"
)
k_skeme.send_keys(skeme)
k_skeme.send_keys(Keys.DOWN)
k_skeme.send_keys(Keys.ENTER)
else:
color_scheme = str(random.randint(1, 29))
driver.find_element_by_id(("downshift-0-item-" + color_scheme)).click()
driver.find_element_by_id("export-menu").click()
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await hell.edit("`Processing..\n75%`")
# Waiting for downloading
await asyncio.sleep(2.5)
color_name = driver.find_element_by_xpath(
"/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input"
).get_attribute("value")
await hell.edit("`Done Dana Done...\n100%`")
file = "./carbon.png"
await hell.edit("`Uploading..`")
await e.client.send_file(
e.chat_id,
file,
caption="`Here's your carbon!` \n**Colour Scheme: **`{}`".format(
color_name),
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
driver.quit()
await hell.delete()
async def carbon_api(e):
""" carbon.now.sh için bir çeşit wrapper """
await e.edit("`İşleniyor...`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Girilen metin, modüle aktarılıyor.
code = quote_plus(pcode) # Çözülmüş url'ye dönüştürülüyor.
await e.edit("`İşleniyor...\nTamamlanma Oranı: 25%`")
if os.path.isfile("./carbon.png"):
os.remove("./carbon.png")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(options=chrome_options)
driver.get(url)
await e.edit("`İşleniyor...\nTamamlanma Oranı: 50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
# driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await e.edit("`İşleniyor...\nTamamlanma Oranı: 75%`")
# İndirme için bekleniyor
while not os.path.isfile("./carbon.png"):
await sleep(0.5)
await e.edit("`İşleniyor...\nTamamlanma Oranı: 100%`")
file = './carbon.png'
await e.edit("`Resim karşıya yükleniyor...`")
await e.client.send_file(
e.chat_id,
file,
caption=
"Bu resim [Carbon](https://carbon.now.sh/about/) kullanılarak yapıldı,\
\nbir [Dawn Labs](https://dawnlabs.io/) projesi.",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./carbon.png')
driver.quit()
# Karşıya yüklemenin ardından carbon.png kaldırılıyor
await e.delete() # Mesaj siliniyor
'id': 'span-compra-tc'
},
'venta': {
'id': 'span-venta-tc'
}
}]
urlTabla2 = "https://cuantoestaeldolar.pe/"
lista2 = [
"Western Union", "CambiaFX", "Cambio Seguro", "Rextie", "Dollar House",
"Cambista Inka", "AComo", "Cambix", "TKambio", "Tu Cambista", "Dólar Sol"
]
options = Options()
options.add_argument("--headless")
class Scraping:
def runDriverManager(self):
self.driver = webdriver.Chrome(ChromeDriverManager().install(),
options=options)
#self.driver = webdriver.Firefox(GeckoDriverManager.install())
self.driver.implicitly_wait(20)
def close(self):
print('Ha detenido el programa')
exit()
def extract(self):
print("run")
from selenium import webdriver
from time import sleep
from selenium.webdriver.chrome.options import Options
from settings import (
DISCORD_CLIENT,
death_path,
recovered_path,
infected_path,
arcgis_url
)
client = discord.Client()
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.binary_location = '/bin/google-chrome'
driver = webdriver.Chrome(options=chrome_options)
def scrape_arcgis():
try:
driver.get(arcgis_url)
sleep(5)
html_source = driver.page_source
dom = etree.HTML(html_source)
deaths = dom.xpath(death_path)[0].text
recovered = dom.xpath(recovered_path)[0].text
infected = dom.xpath(infected_path)[0].text
driver.get("https://www.youtube.com/")
driver.set_window_size(1050, 721)
time.sleep(2)
driver.find_element(By.NAME, "search_query").click()
driver.find_element(By.NAME, "search_query").send_keys(name)
driver.find_element(
By.CSS_SELECTOR,
"#search-icon-legacy > .style-scope:nth-child(1)").click()
time.sleep(4)
j = driver.find_element_by_partial_link_text(name[:len(name) // 2])
link = (j.get_attribute('href'))
# driver.get(link)
links.append(link)
# time.sleep(5)
driver.close()
with open('youtube_input.txt', 'r') as f:
l = f.readlines()
namesList = []
for i in l:
namesList.append(i[:len(i) - 1])
playVideo(namesList)
chrome_options = Options()
chrome_options.add_argument("--incognito")
driver = webdriver.Chrome(chrome_options=chrome_options)
for i in links:
driver.get(i)
time.sleep(10)
async def carbon_api(e):
RED = random.randint(0, 256)
GREEN = random.randint(0, 256)
BLUE = random.randint(0, 256)
OPC = random.random()
hell = await edit_or_reply(e, "⬜⬜⬜⬜⬜")
CARBON = "https://carbon.now.sh/?bg=rgba({R}%2C{G}%2C{B}%2C{O})&t=material&wt=none&l=auto&ds=false&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Fira%20Code&fs=14px&lh=152%25&si=false&es=2x&wm=false&code={code}"
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[7:]:
pcode = str(pcode[7:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code,
R=RED,
G=GREEN,
B=BLUE,
O=OPC,
lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
await hell.edit("⬛⬛⬜⬜⬜")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
await asyncio.sleep(2) # this might take a bit.
# driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
# await asyncio.sleep(5)
await hell.edit("⬛⬛⬛⬜⬜")
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await asyncio.sleep(2) # Waiting for downloading
await hell.edit("⬛⬛⬛⬛⬛")
file = "./carbon.png"
await hell.edit("✅RGB Karbon 2.0 Completed, Uploading Karbon✅")
await e.client.send_file(
e.chat_id,
file,
caption=f"Here's your karbonrgb",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
# Removing carbon.png after uploading
await hell.delete() # Deleting msg
# Generate a certain number of random articles within the list
print(str(len(listOfURLsToBeRandomized)) + ' total articles')
desiredNumberOfArticles = None # have to create variable outside of if statement
if len(listOfURLsToBeRandomized) < numberOfRandomSamples:
print('Using ' + str(len(listOfURLsToBeRandomized)) + ' articles')
desiredNumberOfArticles = listOfURLsToBeRandomized
else:
desiredNumberOfArticles = random.sample(listOfURLsToBeRandomized,
numberOfRandomSamples)
print(len(desiredNumberOfArticles))
# Preparing to start with running chrome driver with Selenium:
chromeDriverPath = '/Users/yanisa/GoogleDrive/Publications_Conferences/Code/2020.CovidMetaphorMetonymyBookChptCollab/chromedriver'
chromeOptions = Options()
chromeOptions.add_argument('--headless')
webDriver = webdriver.Chrome(executable_path=chromeDriverPath,
options=chromeOptions)
listOfInformation = []
with webDriver as driver:
# Set timeout time
wait = WebDriverWait(driver, 20)
driver.implicitly_wait(10)
driver.maximize_window()
print('Opening Selenium...')
index = 1
for eachURL in desiredNumberOfArticles:
time.sleep(randomSleepsFunction())
driver.get(eachURL)
from django.core.management.base import DjangoHelpFormatter
from davinci_crawling.time import mk_datetime
from django.test import RequestFactory
from django.utils import timezone
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from davinci_crawling.entity_diff.diff import make_diff
from solrq import Q
_logger = logging.getLogger("davinci_crawling")
CHROME_OPTIONS = Options()
CHROME_OPTIONS.add_argument("--headless")
CHROME_OPTIONS.add_argument("--no-sandbox")
CHROME_OPTIONS.add_argument("--disable-gpu")
CHROME_OPTIONS.add_argument("--disable-features=NetworkService")
CHROME_OPTIONS.add_argument("--disable-dev-shm-usage")
def get_configuration(crawler_name):
return settings.DAVINCI_CRAWLERS[crawler_name] if crawler_name in settings.DAVINCI_CRAWLERS else {}
class Crawler(metaclass=ABCMeta):
CHROME_DESIREDCAPABILITIES = "chrome_desired_capabilities"
defaults_parser = {
def open_browser(self, browser, proxy=None):
log_level = DesiredCapabilities.CHROME
log_level['loggingPrefs'] = {'browser': 'SEVERE'}
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_experimental_option('prefs', {"profile.managed_default_content_settings.images": 2,
"plugins.plugins_disabled": ["Adobe Flash Player"]})
if proxy:
chrome_options.add_argument('--proxy-server=%s' % proxy)
if browser.lower() == "ie":
webdriver.Ie.__init__(self)
log("I open IE browser.")
elif browser.lower() == "chrome":
webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options)
log("I open Chrome Browser")
elif browser.lower() == "firefox":
webdriver.Firefox.__init__(self)
log("I open Firefox Browser")
elif browser.lower() == 'mobile':
mobile_emulation = {
"deviceMetrics": {"width": 1920, "height": 980, "pixelRatio": 3.0},
"userAgent": "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 "
"(KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53"}
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options)
log("I open Mobile Browser")
def __init__(self, logger, user_data_dir, timeout):
"""Initialize chromedriver for use by all other modules.
Args:
logger: initialized logger object.
user_data_dir: string, directory for chrome data.
timeout: integer, number of seconds to wait for web pages to load.
"""
self.timeout = timeout
data_dir = os.path.join(os.getcwd(), user_data_dir)
self.logger = logger
options = Options()
data_dir_option = '--user-data-dir=%s' % data_dir
options.add_argument(data_dir_option)
options.add_argument('--enable-logging')
options.add_argument('--v=1')
options.add_argument('--lang=en')
options.add_argument('--disable-features=UsePasswordSeparatedSigninFlow')
options.add_experimental_option("windowTypes", ["webview"])
self.driver = webdriver.Chrome(chrome_options=options)
self.action_chain = ActionChains(self.driver)
self.window = {}
self.window['main'] = self.driver.current_window_handle
self.window['logocert'] = None
self.ActionPerformed()
def configure_driver():
opts = Options()
opts.add_argument('--headless')
prefs = {"profile.managed_default_content_settings.images": 2}
opts.add_experimental_option("prefs", prefs)
opts.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36')
driver = webdriver.Chrome(chrome_options=opts, executable_path='D:\soft\chromedriver\chromedriver.exe')
return driver
