def __init__(self, user_agent=None, wait_time=2, fullscreen=False,
options=None, **kwargs):
options = Options() if options is None else options
if user_agent is not None:
options.add_argument("--user-agent=" + user_agent)
if fullscreen:
options.add_argument('--kiosk')
prefs = {"download": {
"default_directory": _DOWNLOAD_PATH,
"directory_upgrade": True,
"extensions_to_open": ""
}}
options.add_experimental_option("prefs", prefs)
self.driver = Chrome(chrome_options=options, **kwargs)
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def __init__(self, scenario, download_directory=default_download_directory,
app_host=None, app_port=None, executable_path=None,
pages=None):
# Contract
must_be(download_directory, "download_directory", (type(None), str))
must_be(app_host, "app_host", (type(None), str))
must_be(app_port, "app_port", (type(None), Number))
must_be(executable_path, "executable_path", (type(None), str))
must_be(pages, "pages", (dict, type(None)))
# TODO[TJ]: This should be implemented as part of the future contracts
# library
if pages is not None:
for key, value in pages.items():
must_be(key, "pages key", str)
must_be(value, "pages value", AppPage)
#
self.scenario = scenario
if download_directory is not None:
options = ChromeOptions()
prefs = {"download.default_directory": download_directory}
options.add_experimental_option('prefs', prefs)
else:
options = None
if app_host is not None:
self.app_host = app_host
if app_port is not None:
self.app_port = app_port
if executable_path is not None:
self.executable_path = executable_path
self.pages = pages
super(ChromeBrowser, self).__init__(
executable_path=self.executable_path, chrome_options=options)
register_exit(self.quit)
def __init__(self, options=None, user_agent=None, wait_time=2,
fullscreen=False, incognito=False, headless=False, **kwargs):
options = Options() if options is None else options
options.add_argument("--disable-infobars")
options.add_argument("--disable-notifications")
if 'accepted_languages' in kwargs:
options.add_experimental_option("prefs", {"intl.accept_languages": kwargs['accepted_languages']})
del kwargs['accepted_languages']
if user_agent is not None:
options.add_argument("--user-agent=" + user_agent)
if incognito:
options.add_argument("--incognito")
if fullscreen:
options.add_argument('--kiosk')
if headless:
options.add_argument('--headless')
options.add_argument('--disable-gpu')
self.driver = Chrome(chrome_options=options, **kwargs)
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super(WebDriver, self).__init__(wait_time)
def initiateChromeWithSize(width, height, pixelRatio=2.0):
mobile_emulation = {
"deviceMetrics": { "width": width, "height": height, "pixelRatio": pixelRatio },
"userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19" }
chrome_options = Options()
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
return webdriver.Chrome(chrome_options=chrome_options)
def get_chrome_driver(self):
opts = Options()
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def setUp(self):
#disble images
chrome_options = Options()
chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2})
self.driver = webdriver.Chrome(chrome_options=chrome_options)
# options=[]
# options.append('--load-images=false')
# self.driver=webdriver.PhantomJS(executable_path='C:\\Users\\VUGON\\Desktop\\phantomjs-2.0.0-windows\\phantomjs-2.0.0-windows\\bin\\phantomjs.exe',service_args=options)
#self.driver.implicitly_wait(30)
self.base_url = "https://m.facebook.com/profile.php?v=info&id=100001150323526&nocollections=1"
def setUp(self):
try:
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.browser = self.input.ui_conf['browser']
self.replica = self.input.param("replica", 1)
self.case_number = self.input.param("case_number", 0)
self.machine = self.input.ui_conf['server']
self.driver = None
self.shell = RemoteMachineShellConnection(self.machine)
#avoid clean up if the previous test has been tear down
if not self.input.param("skip_cleanup", True) \
or self.case_number == 1:
self.tearDown()
self._log_start(self)
self._kill_old_drivers()
#thread for selenium server
if not self._is_selenium_running():
self.log.info('start selenium')
self._start_selenium_thread()
self._wait_for_selenium_is_started()
self.log.info('start selenium session')
if self.browser == 'firefox':
self.log.info("firefox")
# self.driver = webdriver.Remote(desired_capabilities={
# "browserName": "firefox",
# "platform": "MAC",
# })
self.driver = webdriver.Remote(command_executor='http://{0}:{1}/wd/hub'
.format(self.machine.ip,
self.machine.port),
desired_capabilities=DesiredCapabilities.FIREFOX)
elif self.browser == 'chrome':
self.log.info("chrome")
local_capabilities = DesiredCapabilities.CHROME
chrome_options = Options()
chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
local_capabilities.update(chrome_options.to_capabilities())
self.log.info(self.machine.ip)
self.log.info(self.machine.port)
self.driver = webdriver.Remote(desired_capabilities={
"browserName": "chrome",
"platform": "MAC",
})
self.log.info('start selenium started')
self.driver.get("http://127.0.0.1:8091")
self.username = "******"
self.password = "******"
self.driver.maximize_window()
except Exception as ex:
self.input.test_params["stop-on-failure"] = True
self.log.error("SETUP WAS FAILED. ALL TESTS WILL BE SKIPPED")
self.fail(ex)
def get_chrome_driver(self):
"""Setup and return a Chrom[e|ium] browser for Selenium."""
opts = Options()
absp = os.path.abspath
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
opts.add_extension(self.pb_ext_path) # will fail if ext can't be found
if self.browser_bin: # otherwise will use webdriver's default binary
print "Browser binary:", absp(self.browser_bin)
opts.binary_location = self.browser_bin # set binary location
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def __init__(self, username=None, password=None, nogui=False):
if nogui:
self.display = Display(visible=0, size=(800, 600))
self.display.start()
chrome_options = Options()
chrome_options.add_argument('--dns-prefetch-disable')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--lang=en-US')
chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en-US'})
self.browser = webdriver.Chrome('./assets/chromedriver', chrome_options=chrome_options)
self.browser.implicitly_wait(25)
self.logFile = open('./logs/logFile.txt', 'a')
self.logFile.write('Session started - %s\n' \
% (datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
self.username = username or environ.get('INSTA_USER')
self.password = password or environ.get('INSTA_PW')
self.nogui = nogui
self.do_comment = False
self.comment_percentage = 0
self.comments = ['Cool!', 'Nice!', 'Looks good!']
self.photo_comments = []
self.video_comments = []
self.followed = 0
self.follow_restrict = load_follow_restriction()
self.follow_times = 1
self.do_follow = False
self.follow_percentage = 0
self.dont_include = []
self.automatedFollowedPool = []
self.dont_like = ['sex', 'nsfw']
self.ignore_if_contains = []
self.ignore_users = []
self.use_clarifai = False
self.clarifai_secret = None
self.clarifai_id = None
self.clarifai_img_tags = []
self.clarifai_full_match = False
self.like_by_followers_upper_limit = 0
self.like_by_followers_lower_limit = 0
self.aborting = False
def get(browser, download_dir):
"""Returns the WebDriver for the specified browser.
"""
print("Getting Driver")
# clear temp folder
basics.remove_files(download_dir)
# set firefox profile and return instance of HBPDriver
if browser == "firefox":
profile = ff_profile(download_dir)
return HBPDriverFirefox(profile)
elif browser == "chrome":
prefs = {
"profile.default_content_settings.popups": 0,
"download.prompt_for_download": "false",
"download.default_directory": download_dir,
"download.extensions_to_open": "",
}
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--no-sandbox") #This make Chromium reachable
chrome_options.add_argument("--no-default-browser-check") #Overrides default choices
chrome_options.add_argument("--no-first-run")
chrome_options.add_argument("--disable-default-apps")
chrome_options.add_argument("--test-type")
chrome_options.add_experimental_option("prefs", prefs)
PROXY = "bbpproxy.epfl.ch:80"
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": PROXY,
"ftpProxy": PROXY,
"sslProxy": PROXY,
"noProxy": None,
"proxyType": "MANUAL",
"class": "org.openqa.selenium.Proxy",
"autodetect": False
}
chrome_logpath = os.path.join(basics.dir_main(), "browser.log")
driver = webdriver.Chrome(chrome_options=chrome_options,
service_args=["--verbose", "--log-path=%s" % chrome_logpath])
return HBPDriver(driver)
return HBPDriverChrome(chrome_options=chrome_options,
service_args=["--verbose", "--log-path=%s" % chrome_logpath])
else:
raise ValueError("Unsopported browser %s" % browser)
def setup_chromedriver(mode):
# configuring chromedriver:
# my screen res: 1366*768:
# activate for verbose logging:
# service_log_path = "chromedriver.log"
# service_args = ['--verbose']
chrome_options = Options()
if mode is 'primary':
chrome_options.add_argument("--restore-last-session")
chrome_options.add_argument("user-data-dir=master_profile")
chrome_options.add_argument("--window-size={},{}".format(1000, 800))
chrome_options.add_argument("--window-position={},{}".format(1, 1))
elif mode is 'tertiary':
chrome_options.add_argument("--window-size={},{}".format(200, 800))
chrome_options.add_argument("--window-position={},{}".format(1050, 1))
# get rid off: --ignore-certificate-errors command-line flag warning:
chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
chrome_options.add_argument("--disable-infobars") # add command line flags
chrome = Chrome(chrome_options=chrome_options,
# service_args=service_args,
# service_log_path=service_log_path
)
# log in if not already:
if 'readcube.com/session/new' in chrome.current_url and mode is 'primary':
chrome.get('https://app.readcube.com/#/library/collection/~/items')
# logging in via email / password
for i in range(5):
try:
sleep(3)
chrome.find_element_by_id('email').send_keys(user)
chrome.find_element_by_id('password').send_keys(pw)
chrome.find_element_by_class_name('submit-button').click()
except Exception as e:
sleep(1.5)
# print e
else:
break
# returns driver:
return chrome
def chrome_manager(self):
opts = Options()
if self.on_travis: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
opts.add_extension(self.extension_path)
opts.binary_location = self.browser_path
opts.add_experimental_option("prefs", {"profile.block_third_party_cookies": False})
caps = DesiredCapabilities.CHROME.copy()
driver = webdriver.Chrome(chrome_options=opts, desired_capabilities=caps)
try:
yield driver
finally:
driver.quit()
def start(self):
if self.engine == 'chrome':
from selenium.webdriver.chrome.options import Options
options = Options()
options.set_headless(headless=True)
options.add_argument('--disable-gpu')
options.add_argument('--log-level=3')
options.add_argument('--mute-audio')
options.add_experimental_option('prefs', {
'profile.managed_default_content_settings.images': 2,
})
options.binary_location = self.binary_path
driver = webdriver.Chrome(
chrome_options=options,
executable_path=self.driver_path,
)
elif self.engine == 'firefox':
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.firefox.options import Options
options = Options()
options.set_headless(headless=True)
options.add_argument('--disable-gpu')
profile = webdriver.FirefoxProfile()
profile.set_preference('permissions.default.image', 2)
profile.set_preference('media.volume_scale', '0.0')
profile.set_preference('media.autoplay.enabled', False)
binary = FirefoxBinary(self.binary_path)
driver = webdriver.Firefox(
firefox_profile=profile,
firefox_options=options,
firefox_binary=binary,
executable_path=self.driver_path,
log_file=os.devnull,
)
driver.set_window_size(1920, 1080)
self.driver = driver
def set_selenium_local_session(self):
"""Starts local session for a selenium server. Default case scenario."""
if self.aborting:
return self
if self.use_firefox:
if self.firefox_profile_path is not None:
firefox_profile = webdriver.FirefoxProfile(self.firefox_profile_path)
else:
firefox_profile = webdriver.FirefoxProfile()
# permissions.default.image = 2: Disable images load, this setting can improve pageload & save bandwidth
firefox_profile.set_preference('permissions.default.image', 2)
self.browser = webdriver.Firefox(firefox_profile=firefox_profile)
else:
chromedriver_location = './assets/chromedriver'
chrome_options = Options()
chrome_options.add_argument('--dns-prefetch-disable')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--lang=en-US')
# managed_default_content_settings.images = 2: Disable images load, this setting can improve pageload & save bandwidth
# default_content_setting_values.notifications = 2: Disable notifications
# credentials_enable_service & password_manager_enabled = false: Ignore save password prompt from chrome
# 'profile.managed_default_content_settings.images': 2,
# 'profile.default_content_setting_values.notifications' : 2,
# 'credentials_enable_service': False,
# 'profile': {
# 'password_manager_enabled': False
# }
chrome_prefs = {
'intl.accept_languages': 'en-US'
}
chrome_options.add_experimental_option('prefs', chrome_prefs)
self.browser = webdriver.Chrome(chromedriver_location, chrome_options=chrome_options)
self.browser.implicitly_wait(self.page_delay)
self.logFile.write('Session started - %s\n' \
% (datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
return self
class Webpage:
def __init__(self, url, browser='chrome'):
self.driver = None
self.url = url
self.continue_scan = True
if browser == 'chrome':
self.launch_chrome()
else:
self.launch_firefox()
self.driver.get(self.url)
def launch_chrome(self):
self.chrome_options = Options()
self.chrome_options.add_argument('--disable-extensions')
self.chrome_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
self.driver = webdriver.Chrome(chrome_options=self.chrome_options)
self.driver.maximize_window()
def launch_firefox(self):
self.driver = webdriver.Firefox()
self.driver.maximize_window()
@classmethod
def open(cls, url, browser='chrome'):
wbpage = Webpage(url, browser)
return wbpage
def wait_for_url_change(self, function_to_execute):
url_before = self.driver.current_url
function_to_execute()
while not self.did_page_change(url_before, self.driver.current_url):
time.sleep(5)
def did_page_change(self, value_old, value_new):
if value_old == value_new:
return False
else:
return True
def close(self):
self.driver.close()
def init_chrome_driver(timeout=30):
chrome_options = Options()
chrome_options.add_argument("--disable-bundled-ppapi-flash")
chrome_options.add_argument("--disable-plugins-discovery")
chrome_options.add_argument("--disable-webaudio")
chrome_options.add_argument("--mute-audio")
#chrome_options.add_argument("--no-startup-window")
prefs = {}
prefs["plugins.plugins_disabled"] = ["Adobe Flash Player", "Shockwave Flash"]
prefs["profile.managed_default_content_settings.images"] = 2
#prefs["profile.managed_default_content_settings.media_stream"] = 2
chrome_options.add_experimental_option("prefs", prefs)
path = Config.value(SECTION_COMMON, "chromedriver_path")
if path:
log("Chromedriver path: %s" % path, INFO)
driver = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
else:
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.wait = WebDriverWait(driver, timeout)
return driver
def __init__(self, **kwargs):
driver_path = kwargs.get('driver_path', 'chromedriver')
headless = kwargs.get('headless', False)
no_image = kwargs.get('no_image', False)
user_agent = kwargs.get('user_agent', '')
proxy = kwargs.get('proxy', '')
home_page = kwargs.get('home_page', '')
options = Options()
if headless:
options.add_argument('--headless')
if no_image:
option_load_img = {'profile.managed_default_content_settings.images': 2}
options.add_experimental_option('prefs', option_load_img)
if user_agent:
options.add_argument(f'user-agent={user_agent}')
if proxy:
options.add_argument(f'--proxy-server=http://{proxy}')
self.chrome = webdriver.Chrome(executable_path=driver_path, chrome_options=options)
self.chrome.maximize_window()
if home_page:
self.chrome.get(home_page)
class HypothesisSeleniumDriver():
def __init__(self,browser,ext_path=None):
self.ext_path = ext_path
self.driver = None
self.options = Options()
if browser == 'firefox':
self.driver = webdriver.Firefox()
if browser == 'chrome':
if self.ext_path is not None:
self.options.add_extension(self.ext_path)
self.options.add_experimental_option('excludeSwitches', ['test-type','ignore-certificate-errors']) # https://code.google.com/p/chromedriver/issues/detail?id=1081
self.driver = webdriver.Chrome(chrome_options=self.options)
if browser == 'ie':
self.driver = webdriver.Ie()
self.drivername = self.driver.name
#self.driver.implicitly_wait(HypothesisSeleniumTiming().very_long_wait)
self.driver.set_page_load_timeout(HypothesisSeleniumTiming().wait_for_page_load_secs)
self.driver.set_script_timeout(HypothesisSeleniumTiming().wait_for_script_secs)
def test(self):
if sys.argv[1].__eq__('prod'):
self.main_url = "https://blazemeter.com"
elif sys.argv[1].__eq__('qa'):
self.main_url = "https://wwwqa.blazemeter.com"
elif sys.argv[1].__eq__('dev'):
self.main_url = "https://wwwdev.blazemeter.com"
else:
self.main_url = "https://wwwqa.blazemeter.com"
for i in range(2, len(sys.argv)):
driver_param = sys.argv[i]
if driver_param.__eq__("Chrome"):
self.driver = webdriver.Chrome()
self.driver.implicitly_wait(30)
self.driver.set_window_size(1920, 1080)
page_elements["/"] = ['#try_it_now>a', 'TRY IT NOW']
elif driver_param.__eq__("iPhone4"):
mobile_emulation = {"deviceName": "Apple iPhone 4"}
chrome_options = Options()
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
self.driver = webdriver.Chrome(chrome_options=chrome_options)
self.driver.implicitly_wait(30)
page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW']
elif driver_param.__eq__("iPhone5"):
mobile_emulation = {"deviceName": "Apple iPhone 5"}
chrome_options = Options()
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
self.driver = webdriver.Chrome(chrome_options=chrome_options)
self.driver.implicitly_wait(30)
page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW']
elif driver_param.__eq__("Firefox"):
self.driver = webdriver.Firefox()
self.driver.implicitly_wait(30)
self.driver.set_window_size(1920, 1080)
page_elements["/"] = ['#try_it_now>a', 'TRY IT NOW']
elif driver_param.__eq__("Nexus7"):
mobile_emulation = {"deviceName": "Google Nexus 7"}
chrome_options = Options()
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
self.driver = webdriver.Chrome(chrome_options=chrome_options)
self.driver.implicitly_wait(30)
page_elements["/"] = ['#mobile_version_try_it_now', 'TRY IT NOW']
else:
self.driver = webdriver.Firefox()
self.driver.implicitly_wait(30)
self.driver.set_window_size(1920, 1080)
self.over_check(self.driver)
self.driver.quit()
def runSelenium(context, browser_name, browser_dict, cpeLocation, cmdSwitches, op_sys, cache_state, hover=False):
sparrow_controller = browser_dict.get('controller')
cpe_ip = browser_dict.get('ip')
is_windows = op_sys == 'windows'
sparrow_controller.stopSparrow()
# Cleanup leftover .dat files from --bb-on-beer; hopefully this is temporary, waiting on sparrow fix
sparrow_controller.removeLeftoverBBs()
if not sparrow_controller.startWebdriver(exe_loc=WEBDRIVER_LOC[op_sys], whitelist_ip=context.common.jenkinsIp):
return False
driver_options = Options()
driver_options.binary_location = sparrow_controller.getSparrowFullPath(op_sys)
# Account for browser cache state
user_data_warm = sparrow_controller.getSparrowUserDataWarmPath(op_sys)
if cache_state.lower() == 'warm':
driver_options.add_argument('--user-data-dir='+user_data_warm)
if not sparrow_controller.removeDir(dir_path=user_data_warm):
return False
# Initialize new user data dir
remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller)
load_url("http://www.google.com", remote)
time.sleep(20)
try:
remote.quit()
except Exception as e:
logging.exception("Selenium exception caught quiting remote after iteration on cpe %s" % cpe_ip)
with open(context.urlListFile) as f:
context.sitelist = f.read().split('\n')
for s in cmdSwitches:
driver_options.add_argument(s)
driver_options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
# Values used later in analysis
context.urlListLen = len(context.sitelist)
context.numClicksTotal = context.listIterations * context.urlListLen
context.numClicksPerSite = context.listIterations
logging.info("running webdriver test on %s with sparrow switches: %s now.." % (cpe_ip, "; ".join(cmdSwitches)))
logging.info("Running selenium webdriver on %s with sitelist: %s" % (cpe_ip, str(context.sitelist)))
# k = url, v = beerID ; used to verify that the current beer is unique and new
beer_status_dict = {}
# initialize
for site in context.sitelist:
beer_status_dict[site] = None
# Pageloads happen here
site_count = 1
for i in range(context.listIterations):
try:
remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller)
for site in context.sitelist:
try:
if not site:
continue
logging.info('Loading url: %s on cpe: %s' % (site, cpe_ip))
remote.switch_to_window(content_tab)
if site[0:15] == 'sparrow://crash':
res = load_url_and_crash(site, remote)
if cache_state.lower() == 'warm':
sparrow_controller.removeDir(dir_path=user_data_warm)
return res
elif hover:
load_on_hover(site, remote)
else:
load_url(site, remote)
logging.info('%s loaded successfully, number of sites loaded on %s: %s' % (site, cpe_ip, site_count))
site_count += 1
remote.switch_to_window(beerstatus_tab)
# Wait up to 20 seconds for beer ack
num_tries = 20
trys = 0
for i in range(num_tries):
load_url("sparrow://beerstatus/", remote)
if check_beer_status(site, beer_status_dict, remote):
break
trys += 1
time.sleep(1)
if trys == num_tries:
logging.info("No beer ack recieved for %s" % site)
except Exception as e:
logging.exception("Selenium exception caught during site visit on cpe %s" % cpe_ip)
remote, beerstatus_tab, content_tab = start_remote(cpeLocation, driver_options, sparrow_controller)
continue
# Need to quit between iterations in both warm and cold case in order to clean out hint cache
try:
remote.quit()
except Exception as e:
logging.exception("Selenium exception caught quiting remote after iteration on cpe %s" % cpe_ip)
continue
except Exception as e:
logging.exception("Selenium exception caught during iteration on cpe %s" % cpe_ip)
continue
sparrow_controller.stopWebdriver()
sparrow_controller.stopSparrow()
# Clean up
if cache_state.lower() == 'warm':
sparrow_controller.removeDir(dir_path=user_data_warm)
return True
# Per https://sites.google.com/a/chromium.org/chromedriver/capabilities docs:
prefs = {
"savefile.default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads",
"download.default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads",
"download.directory_upgrade" : 1,
"download.prompt_for_download" : 1,
}
prefs = {
"savefile": { "default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads"},
"download": {
"default_directory" : "/Users/vsh/Downloads/Selenium/chrome-downloads",
"directory_upgrade" : 1,
"prompt_for_download": 1,
},
}
chrome_options.add_experimental_option("prefs", prefs)
if False:
browser = webdriver.Chrome(executable_path="/Users/vsh/Downloads/Selenium/chromedriver", chrome_options=chrome_options)
# Most likely plan:
# - Edit profile's Preferences to set the downloads directory, so I don't have to maintain that
# aspect manually.
# - Proceed "one file downloaded at a time.
# - List all files in download directory, kick off file download, then wait until there's one new file. In-progress download files have .crdownload extensions.
# - Find a way to be sure the download ended successfully. See http://stackoverflow.com/questions/559096/check-whether-a-pdf-file-is-valid-python.
# - Pass the one new file to a function that will move and rename it to represent the statement's period.
if False:
SigninBoa(browser)
import sys
import os
import requests
import sqlite3
import module.Database
import module.Produce
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--proxy-server=http://127.0.0.1:8118')
chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2, 'profile.default_content_setting_values.notifications': 2})
facebook_usernames = [""]
facebook_password = ""
#encodes
reload(sys)
sys.setdefaultencoding('utf8')
#setup
module.Database.createDatabase()
conn = sqlite3.connect('facebook.db')
conn.text_factory = str
def loginFacebook(driver, fbun):
driver.implicitly_wait(120)
driver.get("https://www.facebook.com/")
from selenium.webdriver.common.keys import Keys # 键盘按键操作
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait # 等待页面加载某些元素
from selenium.webdriver.chrome.options import Options
from datetime import date, timedelta
import time
import random
import json
import re
# 加启动配置 禁用日志log
chrome_options = Options()
# “–no - sandbox”参数是让Chrome在root权限下跑
chrome_options.add_argument('–no-sandbox')
chrome_options.add_argument('–disable-dev-shm-usage')
chrome_options.add_experimental_option('excludeSwitches',
['enable-automation'])
chrome_options.add_argument('--start-maximized') # 最大化
chrome_options.add_argument('--incognito') # 无痕隐身模式
chrome_options.add_argument("disable-cache") # 禁用缓存
chrome_options.add_argument('log-level=3')
chrome_options.add_argument('disable-infobars')
chrome_options.add_argument('--headless')
url = "https://newids.seu.edu.cn/authserver/login?service=http://ehall.seu.edu.cn/qljfwapp2/sys/lwReportEpidemicSeu/*default/index.do"
enter_campus_url = "http://ehall.seu.edu.cn/ygfw/sys/swmxsqjappseuyangong/*default/index.do#/" # 申请入校网址
dailyDone1 = False
# 创建打卡记录log文件
def writeLog(text):
from django.test import TestCase
import requests
import time
# Create your tests here.
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
chrome_option = Options()
chrome_option.add_argument('--disable-extensions')
# 这里的端口和上面remote-debug对应的端口一致
chrome_option.add_experimental_option('debuggerAddress', '127.0.0.1:9222')
# browser = webdriver.Chrome(executable_path='/Users/vium/chromedriver', chrome_options=chrome_option)
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
browser = webdriver.Chrome(executable_path='/Users/vium/chromedriver', options=options, chrome_options=chrome_option)
browser.get('http://www.nhc.gov.cn/xcs/xxgzbd/gzbd_index.shtml')
content = browser.page_source
# ul = browser.find_element_by_xpath("/html/body/div[3]/div[1]/ul")
time.sleep(3)
# session = requests.session()
# session.headers.update(
# {
# 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'
# }
# )
# r = session.get(url='http://www.nhc.gov.cn/xcs/xxgzbd/gzbd_index.shtml')
class Mydriver:
def __init__(self, noimg=True, nohead=True):
try:
self.options = Options()
if os.path.exists("./chrome/chrome.exe"): # win
self.options.binary_location = "./chrome/chrome.exe"
elif os.path.exists("/opt/google/chrome/chrome"): # linux
self.options.binary_location = "/opt/google/chrome/chrome"
if noimg:
self.options.add_argument('blink-settings=imagesEnabled=false') # 不加载图片, 提升速度
if nohead:
self.options.add_argument('--headless')
self.options.add_argument('--disable-extensions')
self.options.add_argument('--disable-gpu')
self.options.add_argument('--no-sandbox')
self.options.add_argument('--mute-audio') # 关闭声音
self.options.add_argument('--window-size=400,500')
self.options.add_argument('--window-position=800,0')
self.options.add_argument('--log-level=3')
self.options.add_argument('--user-agent={}'.format(user_agent.getheaders()))
self.options.add_experimental_option('excludeSwitches', ['enable-automation']) # 绕过js检测
self.webdriver = webdriver
if os.path.exists("./chrome/chromedriver.exe"): # win
self.driver = self.webdriver.Chrome(executable_path="./chrome/chromedriver.exe",
chrome_options=self.options)
elif os.path.exists("./chromedriver"): # linux
self.driver = self.webdriver.Chrome(executable_path="./chromedriver",
chrome_options=self.options)
elif os.path.exists("/usr/lib64/chromium-browser/chromedriver"): # linux 包安装chromedriver
self.driver = self.webdriver.Chrome(executable_path="/usr/lib64/chromium-browser/chromedriver",
chrome_options=self.options)
elif os.path.exists("/usr/local/bin/chromedriver"): # linux 包安装chromedriver
self.driver = self.webdriver.Chrome(executable_path="/usr/local/bin/chromedriver",
chrome_options=self.options)
else:
self.driver = self.webdriver.Chrome(chrome_options=self.options)
#self.driver.set_window_size(1200, 900)
except:
print("=" * 120)
print("Mydriver初始化失败")
print("=" * 120)
raise
def login(self):
print("正在打开二维码登陆界面,请稍后")
self.driver.get("https://pc.xuexi.cn/points/login.html")
try:
remover = WebDriverWait(self.driver, 30, 0.2).until(
lambda driver: driver.find_element_by_class_name("redflagbox"))
except exceptions.TimeoutException:
print("网络缓慢，请重试")
else:
self.driver.execute_script('arguments[0].remove()', remover)
try:
remover = WebDriverWait(self.driver, 30, 0.2).until(
lambda driver: driver.find_element_by_class_name("header"))
except exceptions.TimeoutException:
print("当前网络缓慢...")
else:
self.driver.execute_script('arguments[0].remove()', remover)
try:
remover = WebDriverWait(self.driver, 30, 0.2).until(
lambda driver: driver.find_element_by_class_name("footer"))
except exceptions.TimeoutException:
print("当前网络缓慢...")
else:
self.driver.execute_script('arguments[0].remove()', remover)
self.driver.execute_script('window.scrollTo(document.body.scrollWidth/2 - 200 , 0)')
try:
WebDriverWait(self.driver, 270).until(EC.title_is(u"我的学习"))
cookies = self.get_cookies()
return cookies
except:
print("扫描二维码超时")
def dd_login(self, d_name, pwd):
__login_status = False
self.driver.get(
"https://login.dingtalk.com/login/index.htm?"
"goto=https%3A%2F%2Foapi.dingtalk.com%2Fconnect%2Foauth2%2Fsns_authorize"
"%3Fappid%3Ddingoankubyrfkttorhpou%26response_type%3Dcode%26scope%3Dsnsapi"
"_login%26redirect_uri%3Dhttps%3A%2F%2Fpc-api.xuexi.cn%2Fopen%2Fapi%2Fsns%2Fcallback"
)
self.driver.find_element_by_id("mobile").click()
self.driver.find_element_by_id("mobile").send_keys(d_name)
self.driver.find_element_by_id('pwd').click()
self.driver.find_element_by_id("pwd").send_keys(pwd)
self.driver.find_element_by_id("loginBtn").click()
pic_src = self.driver.find_element_by_class_name("indentify_content").find_elements_by_tag_name('img')[0].get_attribute('src')
if pic_src !='':
print("遇到验证码!",pic_src)
response = urllib.request.urlopen(pic_src)
#print(response.status)
#print(response.getheaders())
#print(response.read())
cat_b64 = str(base64.b64encode(response.read()))[2:-1]
obj = self.send_captcha_ocr(cat_b64)
if obj['code']==0:
print('验证码识别结果为:' + obj['data']['captcha'])
#self.driver.find_element_by_id("identifyCode").send_keys("123") #这个验证码框默认不可交互
self.go_js("document.querySelector('#identifyCode').value='" + obj['data']['captcha']+"'");
#print(self.driver.find_element_by_id("identifyCode").get_attribute('value'))
self.driver.find_element_by_id("identifybtn").click() #提交按钮
else:
print('验证码识别出错!')
#self.go_js("document.querySelector('.indentify_content').scrollTo();")
#screenImg = "/tmp/yzm.png"
#self.driver.get_screenshot_as_file(screenImg) #屏幕截图
#with open(screenImg,"rb") as f:
# imgdata = base64.b64encode(f.read())
# print(str(imgdata))
#sys.exit(0)
try:
print("登陆中...")
WebDriverWait(self.driver, 30, 0.1).until(lambda driver: driver.find_element_by_class_name("modal"))
self.quit()
__login_status = False
except:
__login_status = True
return __login_status
def get_cookies(self):
cookies = self.driver.get_cookies()
return cookies
def set_cookies(self, cookies):
for cookie in cookies:
self.driver.add_cookie({k: cookie[k] for k in cookie.keys()})
def get_url(self, url):
self.driver.get(url)
def go_js(self, js):
self.driver.execute_script(js)
def quit(self):
self.driver.quit()
def send_captcha_ocr(self,base64):
url = '端口URL'
dict = {'image': base64,'type': '1001','length':4,}
data = urllib.parse.urlencode(dict).encode('utf-8')
headers={'Authorization':'密钥','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'};#(请求headers头有一些供应商是key)
request = urllib.request.Request(url = url,data = data,headers = headers,method = 'POST')
response = urllib.request.urlopen(request)
content = response.read()
if (content):
obj = json.loads(content)
return obj
def navi (email, password):
driver.get("https://open.spotify.com/playlist/7MQ2ATqqdDTnlx3UUDHe5d?si=d3bJe6odSe-b2Ww8obRR_A")
time.sleep(3)
bouton_login = driver.find_element_by_xpath('//*[@id="main"]/div/div[2]/div[1]/header/div[5]/button[2]')
bouton_login.click()
time.sleep(6)
champ_email = driver.find_element(By.ID, "login-username")
champ_email.send_keys(email + Keys.TAB)
champ_password = driver.find_element(By.ID, "login-password")
champ_password.send_keys(password + Keys.ENTER)
time.sleep(5)
bouton_play = driver.find_element_by_xpath('/html/body/div[4]/div/div[2]/div[4]/main/div/div[2]/div/div/div[2]/section/div[3]/div/button[1]')
bouton_play.click()
# mettre la boucle on click jsuque'à ce qu'on soit en mode repeat simple
time.sleep(2)
check_play()
check_repeat()
check_time()
refresh()
options = Options()
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome('/Users/aveyronvictor/Documents/AUTO/chromedriver', options=options)
navi("*****@*****.**", "azerty123")
async def carbon_api(e):
RED = random.randint(0, 256)
GREEN = random.randint(0, 256)
BLUE = random.randint(0, 256)
THEME = [
"3024-night",
"a11y-dark",
"blackboard",
"base16-dark",
"base16-light",
"cobalt",
"dracula",
"duotone-dark",
"hopscotch",
"lucario",
"material",
"monokai",
"night-owl",
"nord",
"oceanic-next",
"one-light",
"one-dark",
"panda-syntax",
"paraiso-dark",
"seti",
"shades-of-purple",
"solarized",
"solarized%20light",
"synthwave-84",
"twilight",
"verminal",
"vscode",
"yeti",
"zenburn",
]
CUNTHE = random.randint(0, len(THEME) - 1)
The = THEME[CUNTHE]
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
await e.edit("⬜⬜⬜⬜⬜")
CARBON = "https://carbon.now.sh/?bg=rgba({R}%2C{G}%2C.{B}%2C1)&t={T}&wt=none&l=auto&ds=false&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Fira%20Code&fs=14px&lh=152%25&si=false&es=2x&wm=false&code={code}"
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[6:]:
pcode = str(pcode[6:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code,
R=RED,
G=GREEN,
B=BLUE,
T=The,
lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
await e.edit("⬛⬛⬜⬜⬜")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath(
"//button[contains(text(),'Export')]").click()
sleep(5) # this might take a bit.
# driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
# sleep(5)
await e.edit("⬛⬛⬛⬜⬜")
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
# sleep(5) #Waiting for downloading
await e.edit("⬛⬛⬛⬛⬛")
file = "./carbon.png"
await e.edit("✅RGB Karbon Completed, Uploading RGB Karbon✅")
await e.client.send_file(
e.chat_id,
file,
caption=
"✅Carbonised by [IndianBot](https://github.com/indianbhaiya/IndianBot/)",
force_document=False,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
# Removing carbon.png after uploading
await e.delete() # Deleting msg
def _esperar(tempo):
i = 0
while i < tempo:
time.sleep(1)
i += 1
# Lidar com notificações do Chrome
option = Options()
option.add_argument("--disable-infobars")
option.add_argument("start-maximized")
option.add_argument("--disable-extensions")
# 1=aceitar / 2=recusar
option.add_experimental_option(
"prefs", {"profile.default_content_setting_values.notifications": 2})
# Abrir página
browser = webdriver.Chrome(
chrome_options=option,
executable_path=
"C:\\Users\\thith\Downloads\chromedriver_win32\chromedriver.exe")
browser.maximize_window()
browser.get('https://dashboard.tawk.to/login')
email = "email"
senha = "senha"
browser.find_element_by_xpath(
"/html/body/div[3]/div[2]/div/form/div[2]/input").send_keys(email)
browser.find_element_by_xpath(
import cx_Oracle #Oracle
import MySQLdb #MySQL
import pymssql #MsSQL
----WebDriver----
driver = webdriver.Chrome(executable_path = "",chrome_options=) #To open a browser
driver.get("http://newtours.demoaut.com/") #To open url
----Browser options----
from selenium.webdriver.chrome.options import Options
chromeOptions=Options()
chromeOptions.add_experimental_option("pref",{"download.default_directory":"C:\NewFolder"})
driver = webdriver.Chrome(chrome_options=chromeOptions)
----headless browser----
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
print(driver.title) #To get the page title
assert "Welcome: Mercury Tours" in driver.title #To verify the title
print(driver.current_url) #To get the current url
print(driver.page_source) #To get the HTML content of the page_source
########################################################################### Main Function ###########################################################################
SUFFIX = {'A':'', 'S':'.S', 'Q':'.Q', 'M':'.M', 'W':'.W'}
REPL = {'A':'', 'S':None, 'Q':'-Q', 'M':'-', 'W':None}
new_item_counts = 0
chrome = None
for g in range(start_file,last_file+1):
if main_file.empty == False:
break
if chrome == None:
options = Options()
options.add_argument("--disable-notifications")
options.add_argument("--disable-popup-blocking")
options.add_argument("ignore-certificate-errors")
options.add_experimental_option("excludeSwitches", ["enable-logging"])
chrome = webdriver.Chrome(ChromeDriverManager().install(), options=options)
chrome.set_window_position(980,0)
logging.info('Reading file: '+NAME+str(g)+' Time: '+str(int(time.time() - tStart))+' s'+'\n')
if g == 1 or g == 2 or g == 8 or g == 9:############################################################ ECB ##################################################################
file_path = data_path+NAME+str(g)+'.csv'
if PRESENT(file_path):
if g == 1 or g == 2:
skip = [0,4]
else:
skip = [3]
FOREX_t = readFile(data_path+NAME+str(g)+'.csv', header_ = [0,1,2], index_col_=0, skiprows_=skip)
#FOREX_t = readFile(data_path+NAME+str(g)+'.csv', header_=[0], index_col_=0)
else:
if g == 1 or g == 8:
url = 'https://sdw.ecb.europa.eu/browse.do?node=9691296'
import time
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import os
from bs4 import BeautifulSoup
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import ElementNotInteractableException
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.common.exceptions import StaleElementReferenceException
url = 'https://www.wsj.com/news/archive/2020/march'
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1080")
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
# url_article = requests(url)
driver.get(url)
sign_in_link = driver.find_element_by_link_text('Sign In')
sign_in_link.click()
username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'username')))
password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, 'password')))
user1 = os.environ.get("USER")
pass1 = os.environ.get("PASS")
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
reload(sys)
sys.setdefaultencoding("utf-8")
DOMAIN = r"http://spiss.91safety.com/"
PAGE = r"http://spiss.91safety.com/entOpen/firstpage.jhtml"
CP = Options()
CP.add_argument("--disable-extensions")
CP.add_argument("--disable-popup-blocking")
CP.add_experimental_option("prefs", {
"credentials_enable_service": False,
"profile": {
"password_manager_enabled": False
}
})
ACCOUNT_PATH = "account2.xlsx"
name = ""
pw = ""
res_name = u""
phone_number = ""
department = u"办公室"
cond = u"已经完成整改"
pos = u"室内"
service_args=[]
service_args.append('--load-images=no') ##关闭图片加载
service_args.append('--disk-cache=yes') ##开启缓存
# * *
# * June 01,2020 *
# * *
# * * * * * * * * * * * * * * * * * * * * * * *
# import packages
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from selenium.webdriver.common.keys import Keys
options = Options()
prefs = {'download.default_directory' : '/home/radia/lrcfiles'}
options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(options=options)
# Authentication
url = "https://www.megalobiz.com/auth/login"
driver.get(url)
print('==== \n')
print("Successfully, Started the Selenium Server! \n ")
email = driver.find_element_by_css_selector('#email_macro_1')
email.send_keys('*******@gmail.com')
password = driver.find_element_by_css_selector('#password_macro_1')
password.send_keys('*****')
login = driver.find_element_by_css_selector('#setting_submit_form_macro_2')
login.click()
time.sleep(4)
def handle(self, *args, **kwargs):
#Open chrome and navigate to webpage
url = 'https://secure.sos.state.or.us/orestar/gotoPublicTransactionSearch.do'
print('url collected')
#Pass arguments to the Chrome Driver
options = Options()
prefs = {"download.default_directory": "./code/downloads"}
options.add_experimental_option("prefs", prefs)
options.add_argument('--disable-dev-shm-usage')
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-extensions')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.set_headless(headless=True)
options.binary_location = "/usr/bin/google-chrome-stable"
print('options collected')
#Initiate Driver
driver = webdriver.Chrome('/code/chromedriver', chrome_options=options)
delay = 3
print('driver created')
#Go to Orestar Transaction Search page
driver.get(url)
#Wait until page is loaded
try:
myElem = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((By.ID, 'header2')))
print("submitting search query")
except TimeoutException:
print("loading took too much time!")
#Grab form and button elements
start = driver.find_element_by_id('cneSearchTranStartDate')
end = driver.find_element_by_id('cneSearchTranEndDate')
search_button = driver.find_element_by_name('search')
#Clear form input by default and input start and end dates
start.clear()
start.send_keys('06/15/2019')
end.clear()
end.send_keys('06/20/2019')
#click search button
search_button.click()
print('searching specified dates')
table = driver.find_element_by_tag_name('tr')
next_btn = driver.find_element_by_name('next')
transactions = []
#while the number of rows is greater than 1 find and click the next button to repeat the following steps
while len(driver.find_elements_by_class_name('evenRow')) > 1:
table = driver.find_element_by_tag_name('tr')
even_rows = table.find_elements_by_class_name('evenRow')
odd_rows = table.find_elements_by_class_name('oddRow')
rows = even_rows + odd_rows
#insert items into a list
for items in rows:
individual = []
odd_chars = ['$', '"']
#insert individual transactions into a list
for cell in items.find_elements_by_xpath('.//td'):
if odd_chars in cell:
individual.append(cell.text)
transactions.append(individual)
try:
wait_for_next = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.NAME, "next")))
finally:
next_btn = driver.find_element_by_name('next')
next_btn.click()
print('next page')
else:
# header = driver.find_element_by_class_name('shadedHeader')
filename = time.strftime('%Y%m%d-%H%M%S')
myFile = open('{}.csv'.format(filename), 'w')
with myFile:
writer = csv.writer(myFile)
writer.writerow([
'Trans ID', 'Original ID', 'Trans Date', 'Filer', 'Payee',
'SubType', 'Amount', 'Agg Amount', 'Payee ID', 'Filer ID',
'Attest by Name', 'Attest Date', 'Review by name',
'Review Date', 'Due Date', 'Occptn ltr date',
'Payment Text', 'Purps Desc', 'Interest', 'check_numb',
'trans_status_ind', 'File by Name', 'File Date',
'Address Book Agent', 'Book Type', 'Title Text',
'Occptn TXT', 'emp_name', 'emp_city', 'emp state',
'employeed', 'self employeed', 'Address line 1',
'Address line 2', 'city', 'state', 'zipcode', 'zipplus',
'county', 'purpose code'
])
writer.writerows(transactions)
print('Records downloaded to {}.csv'.format(filename))
# //TODO: Write test coverage plan
# //TODO: Rewrite scraper to download, rename and move the excel file
#// TODO: Rewrite the loading script to match new fields and load from csv file
#// TODO: Write script to download excel files a week at a time
def get_stock_data(stock_name, out_file_name, from_day, from_month, from_year,
to_day, to_month, to_year):
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
options = Options()
options.add_experimental_option(
"prefs", {
"download.default_directory":
r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True
})
# url = "https://www.bseindia.com/markets/equity/EQReports/StockPrcHistori.aspx?expandable=7&scripcode=532648&flag=sp&Submit=G"
url = "https://www.bseindia.com/markets/equity/EQReports/StockPrcHistori.aspx?scripcode=500247"
chrome_driver = r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks\chromedriver.exe"
driver = webdriver.Chrome(chrome_driver, options=options)
driver.set_page_load_timeout(45)
driver.get(url)
time.sleep(1)
driver.find_element_by_id("ContentPlaceHolder1_smartSearch").clear()
time.sleep(1)
driver.find_element_by_id("ContentPlaceHolder1_smartSearch").send_keys(
stock_name)
time.sleep(2)
time.sleep(1)
drop_down = driver.find_elements_by_css_selector("li.quotemenu a")
for values in drop_down:
values.click()
break
#driver.find_element_by_id("ContentPlaceHolder1_smartSearch").click()
time.sleep(1)
driver.find_element_by_id("ContentPlaceHolder1_txtFromDate").click()
time.sleep(1)
if int(from_year) < 2009:
mySelect = Select(
driver.find_element_by_class_name("ui-datepicker-year"))
time.sleep(1)
mySelect.select_by_visible_text("2009")
time.sleep(1)
mySelect = Select(driver.find_element_by_class_name("ui-datepicker-year"))
time.sleep(1)
mySelect.select_by_visible_text(from_year)
mySelect = Select(driver.find_element_by_class_name("ui-datepicker-month"))
mySelect.select_by_visible_text(from_month)
wait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH,
"//td[@data-handler='selectDay']/a[text()='{}']".format(from_day)
))).click()
time.sleep(1)
driver.find_element_by_id("ContentPlaceHolder1_txtToDate").click()
time.sleep(1)
if int(to_year) < 2009:
mySelect = Select(
driver.find_element_by_class_name("ui-datepicker-year"))
time.sleep(1)
mySelect.select_by_visible_text("2009")
time.sleep(1)
mySelect = Select(driver.find_element_by_class_name("ui-datepicker-year"))
time.sleep(1)
mySelect.select_by_visible_text(to_year)
time.sleep(1)
mySelect = Select(driver.find_element_by_class_name("ui-datepicker-month"))
mySelect.select_by_visible_text(to_month)
wait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH,
"//td[@data-handler='selectDay']/a[text()='{}']".format(to_day)
))).click()
time.sleep(1)
driver.find_element_by_id('ContentPlaceHolder1_btnSubmit').click()
time.sleep(2)
driver.find_element_by_id('ContentPlaceHolder1_btnDownload').click()
time.sleep(2)
driver.quit()
import os
import shutil
path = r"C:\Users\kbhandari\OneDrive - Epsilon\Desktop\Stocks"
os.chdir(path)
filename = max([f for f in os.listdir(path)], key=os.path.getctime)
shutil.move(os.path.join(path, filename), out_file_name)
return True
def open_class(link,
runtime,
class_name,
record_class=False,
debug_port=debug_port,
exec_path=exec_path,
user_data_dir=user_data_dir,
exit_browser=True):
if "https://" not in link:
link = "https://" + link
debug_address = "127.0.0.1:" + str(debug_port)
cmd = "chrome --remote-debugging-port=" + str(
debug_port) + " --user-data-dir=" + user_data_dir
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", debug_address)
if mute:
chrome_options.add_argument("--mute-audio")
try:
print("Trying to connect to driver")
driver = webdriver.Chrome(options=chrome_options,
executable_path=exec_path)
print("Connected")
except selenium.common.exceptions.WebDriverException as e:
print("Couldnt connect\nTrying to open")
if 'cannot connect to chrome at' in str(e):
process = subprocess.Popen(cmd)
print("Trying to connect")
driver = webdriver.Chrome(options=chrome_options,
executable_path=exec_path)
print("Connected")
else:
print("unknown error")
driver.get(link)
driver.maximize_window()
sleep(10)
mic_off = True
cam_off = True
src = driver.page_source
sleep(2)
#Mic and cam off check
count = 0
while True:
if count == 10:
if notifs:
send_alert(custom_msg="Couldnt swtich off camera or mic")
break
if "Turn on microphone" in src and "Turn on camera" in src:
break
else:
count += 1
driver.refresh()
sleep(3)
src = driver.page_source
ask_to_join = False
if "Ask to join" in src:
ask_to_join = True
#join_class
try:
j = driver.find_element_by_xpath(join_now_path)
j.click()
if not ask_to_join:
if notifs:
send_alert(joined_class=True, class_name=class_name)
while ask_to_join:
sleep(
30
) #wait 30 seconds for someone to accept (10*30 seconds = 5 minutes)
src = driver.page_source
if "Ask to join" not in src:
ask_to_join = False
if notifs:
send_alert(joined_class=True, class_name=class_name)
if count == 10 or "You can't join this call" in src:
if notifs:
send_alert(joined_class=False, class_name=class_name)
send_alert(custom_msg="Join request was not accepted")
except:
print("Error\nCouldnt find join button")
if notifs:
send_alert(joined_class=False, class_name=class_name)
#Check for additional join confirmation in case of recording
src = driver.page_source
if "This meeting is being recorded" in src:
try:
r = driver.find_element_by_xpath(record_join_now_path)
r.click()
except:
if notifs:
send_alert(
custom_msg="Joined class but couldnt get past record screen"
)
if minimize_after_join:
driver.minimize_window()
driver.quit()
sleep(runtime) #Sleep for the duration of the class
#Connect again to exit class
try:
print("Connecting to driver")
driver = webdriver.Chrome(options=chrome_options,
executable_path=exec_path)
print("Connected")
except:
print("Couldnt connect to driver")
if notifs:
send_alert(exit_class=False)
#Check if the host ended the meeting before
src = driver.page_source
if "The meeting has ended" in src:
print("Meeting ended by host before end time")
if notifs:
send_alert(custom_msg="Meeting ended by host before end time")
else:
try:
l = driver.find_element_by_xpath(end_class_path)
l.click()
if notifs:
send_alert(exit_class=True, class_name=class_name)
except:
print("Error\nCouldnt find the exit button")
if notifs:
send_alert(exit_class=False, class_name=class_name)
if not exit_browser:
driver.quit()
elif exit_browser:
try:
driver.close()
driver.quit()
except:
try:
process.terminate()
except:
try:
action = ActionChains(driver)
action.key_down(Keys.ALT).key_down(Keys.F4).key_up(
Keys.F4).key_up(Keys.ALT).perform()
except:
print("Error\nCouldnt exit the browser")
if notifs:
send_alert(
custom_msg="Error\nCouldnt exit the browser")
from selenium.webdriver.chrome.options import Options
options = Options()
# 禁止图片和css加载
prefs = {
"profile.managed_default_content_settings.images": 2,
'permissions.default.stylesheet': 2
}
options.add_experimental_option("prefs", prefs)
# 无头模式
options.add_argument('--headless')
def newest_post_clicker():
post = driver.find_element_by_css_selector(
'#main > div.infinite-scroll > div:nth-child(1) > div.content')
title = post.find_element_by_css_selector('h2 > a').text
author = post.find_element_by_css_selector(
'p.text-secondary > small:nth-child(4) > a').text
click_title = driver.find_element_by_link_text(title)
click_title.click()
slider = driver.find_element_by_xpath('//*[@id="tipslider"]')
email = '*****@*****.**'
password = '******'
desired_url = 'https://www.publish0x.com/'
options = Options()
options.add_experimental_option("detach", True)
options.add_argument("--window-size=1920,1080")
## options.add_argument("user-data-dir=/Users/vadim/Library/Application Support/BraveSoftware/Brave-Browser")
options.binary_location = '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'
driver_path = '/usr/local/bin/chromedriver'
driver = webdriver.Chrome(options=options, executable_path=driver_path)
driver.get('https://www.publish0x.com/login')
waitForLoad('//*[@id="email"]')
E_Mail_vak = driver.find_element_by_xpath('//*[@id="email"]')
E_Mail_vak.send_keys(email)
Pass_vak = driver.find_element_by_xpath('//*[@id="password"]')
Pass_vak.send_keys(password)
frame = driver.find_element_by_xpath('//iframe[contains(@src, "recaptcha")]')
driver.switch_to.frame(frame)
Captcha = driver.find_element_by_xpath("//*[@id='recaptcha-anchor']")
Captcha.click()
# coding=utf-8
# @Time : 2019/12/22
# @Author: 星空物语
# @File : demo2.py
# @Description: 自己测试用，无效文件
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
# 使用已打开的Chrome浏览器进行调试
# 首先使用命令打开Chrome浏览器并开启远程调试功能
# chrome.exe --remote-debugging-port=9222 --user-data-dir="D:\code\python\selenium_ui_auto\chrome_temp"
# 手动打开百度首页，以下代码可以执行成功
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
chrome_driver = r"D:\code\python\selenium_ui_auto\driver\chromedriver.exe"
driver = webdriver.Chrome(chrome_driver, chrome_options=chrome_options)
driver.find_element_by_id('kw').send_keys(u'测试工程师小站')
async def carbon_api(e):
""" A Wrapper for carbon.now.sh """
await e.edit("`Processing..`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
await e.edit("`Processing..\n25%`")
if os.path.isfile("./carbon.png"):
os.remove("./carbon.png")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=CHROME_DRIVER,
options=chrome_options)
driver.get(url)
await e.edit("`Processing..\n50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await e.edit("`Processing..\n75%`")
# Waiting for downloading
while not os.path.isfile("./carbon.png"):
await sleep(0.5)
await e.edit("`Processing..\n100%`")
file = './carbon.png'
await e.edit("`Uploading..`")
await e.client.send_file(
e.chat_id,
file,
caption="Made using [Carbon](https://carbon.now.sh/about/),\
\na project by [Dawn Labs](https://dawnlabs.io/)",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./carbon.png')
driver.quit()
# Removing carbon.png after uploading
await e.delete() # Deleting msg
from util.settings import Settings
from util.datasaver import Datasaver
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from util.extractor import extract_information, extract_tag_information
chrome_options = Options()
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--kiosk")
chrome_options.add_argument('--dns-prefetch-disable')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--lang=en-US')
chrome_options.add_argument('--headless')
chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en-US'})
browser = webdriver.Chrome('./assets/chromedriver', chrome_options=chrome_options)
# makes sure slower connections work as well
def slackSendMsg(msg):
webhook_url = 'https://hooks.slack.com/services/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'
slack_data = {'text': '#InstagramCrawl:\n' + msg}
response = requests.post(
webhook_url, data=json.dumps(slack_data),
headers={'Content-Type': 'application/json'}
)
if response.status_code != 200:
raise ValueError(
'Request to slack returned an error %s, the response is:\n%s'
async def carbon_api(e):
""" carbon.now.sh için bir çeşit wrapper """
await e.edit("`İşleniyor...`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Girilen metin, modüle aktarılıyor.
code = quote_plus(pcode) # Çözülmüş url'ye dönüştürülüyor.
await e.edit("`İşleniyor...\nTamamlanma Oranı: 25%`")
if os.path.isfile("./carbon.png"):
os.remove("./carbon.png")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(options=chrome_options)
driver.get(url)
await e.edit("`İşleniyor...\nTamamlanma Oranı: 50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
# driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await e.edit("`İşleniyor...\nTamamlanma Oranı: 75%`")
# İndirme için bekleniyor
while not os.path.isfile("./carbon.png"):
await sleep(0.5)
await e.edit("`İşleniyor...\nTamamlanma Oranı: 100%`")
file = './carbon.png'
await e.edit("`Resim karşıya yükleniyor...`")
await e.client.send_file(
e.chat_id,
file,
caption=
"Bu resim [Carbon](https://carbon.now.sh/about/) kullanılarak yapıldı,\
\nbir [Dawn Labs](https://dawnlabs.io/) projesi.",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./carbon.png')
driver.quit()
# Karşıya yüklemenin ardından carbon.png kaldırılıyor
await e.delete() # Mesaj siliniyor
async def carbon_api(e):
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
await e.edit("`Processing..`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcodee = str(pcode[8:])
if "|" in pcodee:
pcode, skeme = pcodee.split("|")
else:
pcode = pcodee
skeme = None
elif textx:
pcode = str(textx.message)
skeme = None # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
await e.edit("`Making Carbon...\n25%`")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=CHROME_DRIVER,
options=chrome_options)
driver.get(url)
await e.edit("`Be Patient...\n50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath(
'/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]'
).click()
if skeme != None:
k_skeme = driver.find_element_by_xpath(
'/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input'
)
k_skeme.send_keys(skeme)
k_skeme.send_keys(Keys.DOWN)
k_skeme.send_keys(Keys.ENTER)
else:
color_scheme = str(random.randint(1, 29))
driver.find_element_by_id(
("downshift-0-item-" + color_scheme)).click()
driver.find_element_by_id("export-menu").click()
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
driver.find_element_by_xpath(
"//button[contains(text(),'PNG')]").click()
await e.edit("`Processing..\n75%`")
# Waiting for downloading
sleep(2.5)
color_name = driver.find_element_by_xpath(
'/html/body/div[1]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]/input'
).get_attribute('value')
await e.edit("`Done...\n100%`")
file = './carbon.png'
await e.edit("`Uploading..`")
await e.client.send_file(
e.chat_id,
file,
caption=
"<< `Here's your carbon!` \n**Carbonised by** SudoBot>>\n**Colour Scheme: **`{}`"
.format(color_name),
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./Carbon.png')
driver.quit()
# Removing carbon.png after uploading
await e.delete() # Deleting msg
def getDatasets(chrome_driver ="/home/twesleyb/src/chromedriver.exe",
''' Collect all available datasets from cells.ucsc.edu. '''
## Default parameters.
LOG_LEVEL = 3
META_EXTENSION = ".tsv"
DATA_EXTENSION = ".tsv.gz"
BASEURL = "https://cells.ucsc.edu/"
CHROME_DRIVER = "/home/twesleyb/src/chromedriver.exe"
chrome_driver = CHROME_DRIVER
# Chrome options and path to chromedriver.
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument('log-level=' + str(LOG_LEVEL))
chrome_options.add_experimental_option('excludeSwitches',['enable-logging'])
# Start headless chromedriver session.
driver = webdriver.Chrome(chrome_driver,options=chrome_options)
driver.get(BASEURL)
xpath = "//a[@role='button'][@class='list-group-item tpDatasetButton']"
buttons = driver.find_elements_by_xpath(xpath)
data = list()
for button in buttons:
button.click()
sleep(1)
tabs = driver.find_elements_by_class_name('tpDatasetTab')
def test_fac():
cwd = os.getcwd()
chrome_logpath = os.path.join(cwd, "browser.log")
print("CWD: %s log: %s"%(cwd, chrome_logpath))
prefs = {
"profile.default_content_settings.popups":0,
"download.prompt_for_download": "false",
"download.default_directory": cwd,
}
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
#chrome_options.add_argument("--headless")
#chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_experimental_option("prefs", prefs)
PROXY = "bbpproxy.epfl.ch:80"
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy":PROXY,
"ftpProxy":PROXY,
"sslProxy":PROXY,
"noProxy":None,
"proxyType":"MANUAL",
"class":"org.openqa.selenium.Proxy",
"autodetect":False
}
if True:
driver = webdriver.Chrome(chrome_options=chrome_options, service_args=["--verbose", "--log-path=%s"%chrome_logpath])
else:
driver = webdriver.Remote(
command_executor='http://bbpsrvc22.bbp.epfl.ch:4444/wd/hub',
#desired_capabilities=webdriver.DesiredCapabilities.CHROME)
desired_capabilities=chrome_options.to_capabilities())
driver.get("https://www.google.com/ncr")
elem = driver.find_element_by_xpath("//input[@id='lst-ib']")
elem.send_keys("blue brain project")
elem.submit()
text = driver.find_element_by_xpath("//div[@id='resultStats']").text
print(text)
driver.get("https://ufile.io/yo4y6")
time.sleep(10)
elem = driver.find_element_by_xpath("//a[@class='free-download btn btn-default btn-lg standard-button free-download-button']")
driver.execute_script("arguments[0].click();", elem)
time.sleep(10)
# Open the downloaded zip file
import glob
import zipfile
created_files = glob.glob(os.path.join(cwd, "result.zip"))
print(created_files)
assert_equal(len(created_files), 1)
created_file = created_files[0]
z = zipfile.ZipFile(created_file)
for f in z.namelist():
print(f)
filepath = os.path.join(dirpath, log)
if not os.path.isfile(filepath):
with open(filepath, 'w') as f:
pass
# init logger
with open('logging.yaml', 'r') as file:
config = yaml.load(file, Loader=yaml.FullLoader)
logging.config.dictConfig(config)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# init driver
driverpath = os.path.join(os.path.dirname(__file__), 'chromedriver')
options = Options()
options.add_experimental_option('detach', True)
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
driver = webdriver.Chrome(executable_path=driverpath, options=options)
def login():
try:
driver.get('')
driver.find_element(By.NAME, 'id_petugas').send_keys('')
driver.find_element(By.NAME, 'password').send_keys('')
except (NoSuchElementException) as e:
logger.exception(e)
driver.quit()
async def carbon_api(e):
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
await e.edit("`Processing..`")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
global CARBONLANG
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
await e.edit("`Meking Carbon...\n25%`")
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=CHROME_DRIVER,
options=chrome_options)
driver.get(url)
await e.edit("`Be Patient...\n50%`")
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath(
"//button[contains(text(),'Export')]").click()
#driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
#driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await e.edit("`Processing..\n75%`")
# Waiting for downloading
sleep(2.5)
await e.edit("`Done Dana Done...\n100%`")
file = './carbon.png'
await e.edit("`Uploading..`")
await e.client.send_file(
e.chat_id,
file,
caption=
"<< Here's your carbon, \n Carbonised by [SimbhaUserbot](https://www.github.com/NikhilNGY/SimbhaUserbot)>> ",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./SimbhaUserbot.jpg')
driver.quit()
# Removing carbon.png after uploading
await e.delete() # Deleting msg
async def carbon_api(e):
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
await e.edit("Processing...")
CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('--disable-gpu')
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
await e.edit("Processing 30%")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_path
}
}
command_result = driver.execute("send_command", params)
driver.find_element_by_xpath(
"//button[contains(text(),'Export')]").click()
sleep(5) # this might take a bit.
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
sleep(5)
await e.edit("Processing 50%")
driver.find_element_by_xpath(
"//button[contains(text(),'PNG')]").click()
sleep(5) #Waiting for downloading
await e.edit("Processing 90%")
file = './carbon.png'
await e.edit("Done!!")
await e.client.send_file(
e.chat_id,
file,
caption="《《CARBON by [BorgBot](github.com/ArnabXD/BorgBot)》》",
force_document=False,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./carbon.png')
# Removing carbon.png after uploading
await e.delete() # Deleting msg
async def carbon_api(e):
if not e.text[0].isalpha() and e.text[0] not in ("/", "#", "@", "!"):
""" A Wrapper for carbon.now.sh """
hmm = await e.reply("📛📛📛📛📛")
CARBON = 'https://carbon.now.sh/?bg=rgba(239%2C40%2C44%2C1)&t=one-light&wt=none&l=application%2Ftypescript&ds=true&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Hack&fs=14px&lh=143%25&si=false&es=2x&wm=false&code={code}'
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[8:]:
pcode = str(pcode[8:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.GOOGLE_CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('--disable-gpu')
prefs = {'download.default_directory': './'}
chrome_options.add_experimental_option('prefs', prefs)
await hmm.edit("🔘🔘📛📛📛")
driver = webdriver.Chrome(
executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = './'
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {'cmd': 'Page.setDownloadBehavior', 'params': {
'behavior': 'allow', 'downloadPath': download_path}}
driver.execute("send_command", params)
driver.find_element_by_xpath(
'//*[@id="__next"]/main/div[3]/div[2]/div[1]/div[1]/div/span[2]').click()
driver.find_element_by_id("export-menu").click()
# driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
sleep(5) # this might take a bit.
driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
sleep(5)
await hmm.edit("🔘🔘🔘📛📛")
driver.find_element_by_xpath(
"//button[contains(text(),'PNG')]").click()
sleep(5) # Waiting for downloading
await hmm.edit("🔘🔘🔘🔘🔘")
file = './carbon.png'
await hmm.edit("☣️Karbon2 Completed, Uploading Karbon☣️")
await e.client.send_file(
e.chat_id,
file,
caption="Karbon2 by [@PhycoNinja13b](https://github.com/Phyco-Ninja/UniNinja)",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove('./carbon.png')
# Removing carbon.png after uploading
await hmm.delete() # Deleting msg
async def carbon_api(e):
""" A Wrapper for carbon.now.sh """
cat = await edit_or_reply(e, "🌚🌚🌚🌚🌚")
CARBON = "https://carbon.now.sh/?bg=rgba(29%2C40%2C104%2C1)&t=one-light&wt=none&l=application%2Ftypescript&ds=true&dsyoff=20px&dsblur=68px&wc=true&wa=true&pv=56px&ph=56px&ln=false&fl=1&fm=Hack&fs=14px&lh=143%25&si=false&es=2x&wm=false&code={code}"
CARBONLANG = "en"
textx = await e.get_reply_message()
pcode = e.text
if pcode[6:]:
pcode = str(pcode[6:])
elif textx:
pcode = str(textx.message) # Importing message to module
code = quote_plus(pcode) # Converting to urlencoded
url = CARBON.format(code=code, lang=CARBONLANG)
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = Config.CHROME_BIN
chrome_options.add_argument("--window-size=1920x1080")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
prefs = {"download.default_directory": "./"}
chrome_options.add_experimental_option("prefs", prefs)
await cat.edit("🌝🌝🌚🌚🌚")
driver = webdriver.Chrome(executable_path=Config.CHROME_DRIVER,
options=chrome_options)
driver.get(url)
download_path = "./"
driver.command_executor._commands["send_command"] = (
"POST",
"/session/$sessionId/chromium/send_command",
)
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": download_path
},
}
driver.execute("send_command", params)
driver.find_element_by_xpath("//button[contains(text(),'Export')]").click()
await asyncio.sleep(2)
await cat.edit("🌝🌝🌝🌚🌚")
# driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
await asyncio.sleep(2) # Waiting for downloading
await cat.edit("🌝🌝🌝🌝🌝")
file = "./carbon.png"
await cat.edit("✅Karbon4 Completed, Uploading Karbon✅")
await e.client.send_file(
e.chat_id,
file,
caption=f"Here's your Karbon4 ",
force_document=True,
reply_to=e.message.reply_to_msg_id,
)
os.remove("./carbon.png")
# Removing carbon.png after uploading
await cat.delete() # Deleting msg
import os
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from util.cli_helper import get_all_tags
from util.extractor import extract_information
from util.extractor import get_usernames_from_tag
chrome_options = Options()
chrome_options.add_argument('--dns-prefetch-disable')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--lang=en-US')
# chrome_options.add_argument('--headless')
chrome_options.add_experimental_option('prefs',
{'intl.accept_languages': 'en-US'})
browser = webdriver.Chrome('./assets/chromedriver',
chrome_options=chrome_options)
# makes sure slower connections work as well
browser.implicitly_wait(25)
def rotate_number(prefix, suffix):
i = 1
while os.path.exists(prefix + str(i) + suffix):
i += 1
return i
def getAllUsers(path):
display = Display(visible=0, size=(800, 800))
display.start()
options = Options()
options.add_argument("--headless")
options.add_argument('--no-sandbox')
options.add_argument("--window-size=1920,1080")
options.add_argument('--disable-dev-shm-usage')
options.add_argument("--disable-extensions")
options.add_argument("--disable-gpu")
options.add_argument("--enable-javascript")
options.add_argument("disable-infobars")
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--disable-browser-side-navigation")
#====================================================================================================#
capa = DesiredCapabilities.CHROME
capa["pageLoadStrategy"] = "eager"
driver = webdriver.Chrome(ChromeDriverManager().install(),
options=options,
desired_capabilities=capa)
print("Headless Chrome Initialized")
params = {'behavior': 'allow', 'downloadPath': r''}
driver.execute_cdp_cmd('Page.setDownloadBehavior', params)
download_dir = '/Downloads'
class DGBRXtractor:
def __init__(self):
pass
def wb_start(self):
# Ignorar avisos
warnings.filterwarnings('ignore')
# Configuração do webdriver
self.chrome_options = Options()
self.chrome_options.add_argument('--headless')
self.chrome_options.add_experimental_option('excludeSwitches',
['enable-logging'])
self.browser = webdriver.Chrome(
executable_path='./tools/webdrivers/chromedriver_linux.exe',
options=self.chrome_options)
'''
Este método foi desenvolvido para realizar o download de todos os arquivos do conjunto de dados a partir da sua url.
url -> variável do tipo string | Contém a url da página do dataset que possui os arquivos a serem baixados.
ext -> variável do tipo lista | Lista de strings com as extensões dos arquivos que deseja-se baixar. Por exemplo: ['.csv', '.xlsx', '.pdf']. O valor padrão é somente ['.csv'].
'''
def get_files_by_ds_url(self, url, ext=['.csv']):
#Carregar configurações do WebBrowser
self.wb_start()
# Carregamento da página do dataset
print('Carregando [{}]'.format(url))
try:
init = time.time()
self.browser.get(url)
end = time.time()
print('Página carregada com sucesso ({} s).'.format(end - init))
except:
print(
'Ocorreu um erro ao carregar a página, tente novamente por favor.'
)
self.browser.close()
sys.exit()
# Obter o nome do dataset a partir da url da página
dataset_id = self.browser.current_url.split('/')[-1]
# Selecionar os elementos HTML <li> que possuem a classe "resource-item" | file_elements é uma lista
file_elements = self.browser.find_elements_by_css_selector(
'li.resource-item')
# Iterar por todos os elementos armazenados em file_elements
for element in file_elements:
# Obter o formato do arquivo a ser baixado. Esse elemento se encontra no atributo "data-format" da tag <span> que fica dentro dos <li> que selecionamos anteriormente
file_format = '.' + element.find_element_by_css_selector(
'span').get_attribute('data-format')
# Transformar arquivos que são denotados como .zip+css ou .zip+xlsx e outros apenas em .zip
if file_format.startswith('.zip'):
file_format = '.zip'
# Verificar se o formato de arquivo raspado está presente na lista de extensões que desejamos baixar
if file_format in ext:
# Gerar um nome para o arquivo a ser salvo. Aqui foi utilizado o atributo "title" das tags <a> que contém o link para o detalhamento do dataset, que possuem classe "heading"
file_name = unidecode(
element.find_element_by_css_selector('a.heading').
get_attribute('title').lower().strip().replace(
' ', '-').replace('/', '-'))
# Formatar o nome do arquivo para adicionar a extensão
if not file_name.endswith(file_format):
file_name += file_format
# Obter o link para download do arquivo. Ele se encontra no atributo "href" das tags <a> que possuem classe "resource-url-analytics"
file_url = element.find_element_by_css_selector(
'a.resource-url-analytics').get_attribute('href')
# Verificar se já existe uma pasta no diretório de execução com a identificação do dataset que pegamos lá em cima, caso não exista ela será criada
if not os.path.exists(dataset_id):
os.makedirs(dataset_id)
# Baixar o arquivo utilizando a biblioteca requests
print('Downloading {}'.format(file_name))
req = requests.get(file_url)
# Definir o local e o nome que o arquivo será salvo
file_path = dataset_id + '/' + file_name
# Gravar o arquivo
with open(file_path, mode='wb') as f:
f.write(req.content)
f.close()
# Fechar o browser
self.browser.close()
'''
Este método verifica se existe algum tipo de paginação na página atual.
'''
def have_pagination(self):
try:
num_pages = self.browser.find_elements_by_css_selector(
'div.pagination li')[-2].text
return num_pages
except:
return 1
'''
Este método cria um objeto .json contendo todas as organizações presentes no DADOS.GOV.BR,
bem como seus URLS.
'''
def update_organization_dictionary(self):
print('Atualizando dicionário de organizações do DADOS.GOV.BR')
#Carregar configurações do WebBrowser
self.wb_start()
#Carregar a página inicial das organizações
url = 'https://dados.gov.br/organization?q=&sort=&page='
try:
init = time.time()
self.browser.get(url + '1')
end = time.time()
#print('Página carregada com sucesso ({} s).'.format(end-init))
except:
print(
'Ocorreu um erro ao carregar a página para obter elementos de paginação, tente novamente por favor.'
)
self.browser.close()
sys.exit()
#Verificar paginação
pages = int(self.have_pagination())
#Variável que irá receber o dicionário
organizations = {}
#Criar looping por todas as páginas
for page in range(1, pages + 1):
#Carregar a página
try:
init = time.time()
completed_url = url + str(page)
self.browser.get(completed_url)
end = time.time()
print('Página [{}] carregada com sucesso ({} s).'.format(
completed_url, end - init))
except:
print(
'Ocorreu um erro ao carregar a página, tente novamente por favor.'
)
self.browser.close()
sys.exit()
#Coletar as informações SIGLA, NOME e URL
cards = self.browser.find_elements_by_css_selector(
'li.media-item a')
for card in cards:
org_name = card.get_attribute('title')
org_initials = org_name.split(' ')[-1]
org_url = card.get_attribute('href')
organizations[org_initials] = {
'name': org_name,
'url': org_url
}
with open('org_dictionary.json', 'wb') as f:
f.write(json.dumps(organizations))
f.close()
print('Atualização realizada com êxito.')
'''
Este método tem como finalidade recolher todos os links de datasets de uma organização,
por meio do link da página desta organização no DADOS.GOV.BR.
org_url -> variável do tipo string | URL da página da organização no DADOS.GOV.BR
'''
def get_ds_urls_by_organization_url(self, org_url):
print('Carregando URLs dos conjuntos de dados...')
#Carregar as configurações do WebBrowser
self.wb_start()
#Carregar a página
self.browser.get(org_url)
#Verificar paginação
pages = int(self.have_pagination())
#Variável que irá armazenar os links
datasets = []
for page in range(1, pages + 1):
#Carregar a página
url = org_url + '?page=' + str(page)
#print('carregando... ' + url)
self.browser.get(url)
#Obter os URLs do dataset
ds_url_list = self.browser.find_elements_by_css_selector(
'h3.dataset-heading a')
#print(ds_url_list)
#Adicionando os links na lista
for url in ds_url_list:
datasets.append(url.get_attribute('href'))
print('Datasets carregados com sucesso.')
return datasets
def create_training_data():
"""
Get articles and store with share price change over article date.
"""
# Run/return CLI control
run_script = ""
while run_script != "y":
run_script = input(
"\nWARNING: If ..\data\\training_data.csv exists, this script will replace it."
" Check that ../data/RR.L.csv is up to date with "
"uk.reuters.com/companies/RR.L/news.\nContinue? [y/n]\n")
if run_script == "n":
print("Returning to main menu...")
return
elif run_script != "y":
print("Invalid input")
print(
'\n######################\nCreating training data\n######################'
)
# Start webdriver and get the article list page
print('\nStarting webdriver...')
opts = Options()
opts.set_headless()
opts.add_argument('log-level=3') # Supress log (not fatal errors)
opts.add_experimental_option('excludeSwitches',
['enable-logging']) # As above
driver = webdriver.Chrome(options=opts)
driver.implicitly_wait(120)
URL = 'https://uk.reuters.com/companies/RR.L/news'
driver.get(URL)
# Close cookie pop up
print('\nClosing cookie popup...')
button = driver.find_element_by_id("_evidon-banner-acceptbutton")
button.click()
# Scroll down to load required no. of articles and store their anchor
# tags
print('\nFinding articles...')
articles = driver.find_elements_by_class_name(
"MarketStoryItem-headline-2cgfz")
no_of_articles = len(articles)
# Multiples of 20 (there are 20 articles per page)
while no_of_articles < 240:
driver.find_element_by_tag_name("body").send_keys(Keys.PAGE_DOWN)
articles = driver.find_elements_by_class_name(
"MarketStoryItem-headline-2cgfz")
no_of_articles = len(articles)
print("\nNumber of articles found: {}".format(no_of_articles))
# Prep price data csv for reading
print('\nGetting share price data...')
prices_csv = open('../data/RR.L.csv', 'r', newline='')
prices_reader = csv.reader(prices_csv, delimiter=',')
next(prices_reader, None) # Skip header
# Create list of closing prices and dates
price_list = []
for record in prices_reader:
if record[4] != 'null':
record_list = [datetime.strptime(record[0], '%Y-%m-%d'), record[4]]
price_list.append(record_list)
prices_csv.close()
print('\nNumber of closing price records found: {}'.format(
len(price_list)))
print('\nCreating training data [memory usage limit 95%]...')
# Delete existing training data if it exists
try:
os.remove('../data/training_data.csv')
except OSError:
pass
# Create training data csv and prepare for writing
training_data_csv = open('../data/training_data.csv', 'a', newline='')
training_data_writer = csv.writer(training_data_csv)
# Loop to extract relevent text from each article
data_list = []
for idx, article in enumerate(articles):
# Open article link in new tab and switch to it
article.send_keys(Keys.CONTROL + Keys.RETURN)
driver.switch_to.window(driver.window_handles[1])
# Find and get text from relevent elements, store as string
text_list = []
text_list.append(
driver.find_element_by_class_name(
'ArticleHeader-headline-NlAqj').text)
for para in driver.find_elements_by_class_name(
'ArticleBody-para-TD_9x'):
text_list.append(para.text)
text = ' '.join(text_list)
# Find and get publish date
date = driver.find_element_by_class_name(
'ArticleHeader-date-Goy3y').text
date = datetime.strptime(date, '%B %d, %Y')
# Find index of price record on date of publish, or closest date after
price_date_index = np.array([
record[0] - date if record[0] >= date else timedelta.max
for record in price_list
]).argmin(0)
# Find percentage change from previous close price to next close price
perc_change = 100 * ((float(price_list[price_date_index + 1][1]) -
float(price_list[price_date_index - 1][1])) /
float(price_list[price_date_index - 1][1]))
# Assign article classification based on percentage change
if abs(perc_change) < 1:
classification = 'neutral'
elif perc_change < 0:
classification = 'decrease'
else:
classification = 'increase'
# Append article record to full data list
record = [text, date, perc_change, classification]
data_list.append(record)
print('\nNumber of articles processed: {}'.format(idx + 1))
# Monitor memory usage to ensure does not exceed set limit of 95%
# If memory exceeds 95% save to disk and check again
# If continues to exceed 95% close dirver and exit program
memory_usage = psutil.virtual_memory().percent
print("Current memory usage: {}%".format(memory_usage))
if memory_usage > 95.0:
print("Current memory usage exceeds 95%. Saving data to disk..")
training_data_writer.writerows(data_list)
data_list = []
if memory_usage > 95.0:
print("Memory usage still exceeds 95%. Exiting...")
driver.close()
driver.switch_to.window(driver.window_handles[0])
driver.close()
exit()
# Close tab and switch back to article list page
driver.close()
driver.switch_to.window(driver.window_handles[0])
# Once all articles processed close driver
driver.close()
# Save data list if has not been saved due to exceeding memory limit
if len(data_list) > 0:
training_data_writer.writerows(data_list)
print('\nTraining data created. Number of records in training data: {}'.
format(no_of_articles))
training_data_csv.close()
print('\nReturning to main menu...')
return
from wordcloud import WordCloud, STOPWORDS
import module.Database
global uid
global postcount
#NR of posts downloaded per user
postcount=200
#facebook dummy account login info
facebook_username = "******"
facebook_password = "******"
#browser settings images disabled
chrome_options = Options()
chrome_options.add_experimental_option( "prefs", {'profile.default_content_settings.images': 2})
driver = webdriver.Chrome(chrome_options=chrome_options)
reload(sys)
sys.setdefaultencoding('utf8')
module.Database.createDatabase()
conn = sqlite3.connect('facebook.db')
conn.text_factory = str
def normalize(s):
if type(s) == unicode:
return s.encode('utf8', 'ignore')
else:
return str(s)
def loginFacebook(driver):
driver.implicitly_wait(120)
def get_max_y_value(y_values_list):
maximum_y_value = y_values_list[0]
for tmp_y_value in y_values_list[1:]:
if maximum_y_value < tmp_y_value:
maximum_y_value = tmp_y_value
return maximum_y_value
chrome_options = Options()
chrome_options.add_experimental_option('prefs', {
'credentials_enable_service': False,
'profile': {
'password_manager_enabled': False
}
})
flights_numbers = []
dates = []
driver = webdriver.Chrome(os.getcwd() + '\chromedriver.exe', chrome_options=chrome_options)
# First sign-in
driver.get('https://www.flightradar24.com')
element = driver.find_element_by_css_selector(".premium-menu-title.premium-menu-title-login")
def setUp(self):
chromium_path = '/usr/bin/chromium'
chromedriver_path = './chromedriver'
opts = Options()
opts.binary_location = chromium_path
opts.add_experimental_option("prefs", {
"download.default_directory": "./test_downloaded",
"helperApps.neverAsk.saveToDisk": "octet/stream",
"directory_upgrade": True,
"profile": {"default_content_settings": {"multiple-automatic-downloads": 1}}
})
self.driver = webdriver.Chrome(chrome_options=opts, executable_path=chromedriver_path)
#self.driver = webdriver.Firefox()
url = os.getenv('TEST_URL', config.url)
self.driver.get(url)
self.__load_pages()
# Sets up by redirecting to login page
main_page = page.MainPage(self.driver)
main_page.click_login_link()
def __init__(self, logger, user_data_dir, timeout):
"""Initialize chromedriver for use by all other modules.
Args:
logger: initialized logger object.
user_data_dir: string, directory for chrome data.
timeout: integer, number of seconds to wait for web pages to load.
"""
self.timeout = timeout
data_dir = os.path.join(os.getcwd(), user_data_dir)
self.logger = logger
options = Options()
data_dir_option = '--user-data-dir=%s' % data_dir
options.add_argument(data_dir_option)
options.add_argument('--enable-logging')
options.add_argument('--v=1')
options.add_argument('--lang=en')
options.add_argument('--disable-features=UsePasswordSeparatedSigninFlow')
options.add_experimental_option("windowTypes", ["webview"])
self.driver = webdriver.Chrome(chrome_options=options)
self.action_chain = ActionChains(self.driver)
self.window = {}
self.window['main'] = self.driver.current_window_handle
self.window['logocert'] = None
self.ActionPerformed()
def open_browser(self, browser, proxy=None):
log_level = DesiredCapabilities.CHROME
log_level['loggingPrefs'] = {'browser': 'SEVERE'}
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_experimental_option('prefs', {"profile.managed_default_content_settings.images": 2,
"plugins.plugins_disabled": ["Adobe Flash Player"]})
if proxy:
chrome_options.add_argument('--proxy-server=%s' % proxy)
if browser.lower() == "ie":
webdriver.Ie.__init__(self)
log("I open IE browser.")
elif browser.lower() == "chrome":
webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options)
log("I open Chrome Browser")
elif browser.lower() == "firefox":
webdriver.Firefox.__init__(self)
log("I open Firefox Browser")
elif browser.lower() == 'mobile':
mobile_emulation = {
"deviceMetrics": {"width": 1920, "height": 980, "pixelRatio": 3.0},
"userAgent": "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 "
"(KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53"}
chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
webdriver.Chrome.__init__(self, desired_capabilities=log_level, chrome_options=chrome_options)
log("I open Mobile Browser")
def configure_driver():
opts = Options()
opts.add_argument('--headless')
prefs = {"profile.managed_default_content_settings.images": 2}
opts.add_experimental_option("prefs", prefs)
opts.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36')
driver = webdriver.Chrome(chrome_options=opts, executable_path='D:\soft\chromedriver\chromedriver.exe')
return driver
