def download_video(driver: WebDriver, buttons: List[WebElement], outputFileName: str, quality: str="360p"):
b: WebElement
for b in buttons:
if quality.lower() in b.text.lower():
btn: WebElement = b
hover(driver, btn)
btn.click()
### TODO: Wait for doc to load, then download the video using the btn href link.
#driver.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.TAB)
driver.switch_to.window(driver.window_handles[1])
download_button_xpath = "/html/body/div/center[2]/a"
download_button: WebElement = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, download_button_xpath))
)
video_file_link: str = download_button.get_attribute("href")
### TODO: switch back to main window
while len(driver.window_handles) > 1:
driver.switch_to.window(driver.window_handles[1])
driver.close()
driver.switch_to.window(driver.window_handles[0])
write_file_from_link(video_file_link, outputFileName)
def say_hello(driver: RemoteWebDriver, url: str) -> None:
"""say hello to HR"""
js_code = 'window.open("%s")' % url
driver.execute_script(js_code)
logger.info(driver.current_window_handle)
time.sleep(30)
driver.close()
def data_mining(driver: WebDriver):
"""
Main function providing flow to the app. Next page button is different on first page then on others so when page is
not having the next page button the button's value is changed to boolean which means last article page. Program goes
through last page's articles and breaks loop.
:param driver: WebDriver
:return:
"""
driver.get("https://teonite.com/blog/")
driver.maximize_window()
next_page_button = driver.find_element_by_css_selector(
"ul.pagination-list>li.blog-button.post-pagination>a")
while True:
main_window = driver.window_handles[0]
articles = driver.find_elements_by_class_name("post-container")
for article in articles:
article_title = get_article_title(article)
open_article_in_new_tab(driver, article, article_title)
save_article_in_db(driver, article_title)
driver.close()
driver.switch_to.window(main_window)
if not isinstance(next_page_button, bool):
next_page_button.click()
try:
next_page_button = driver.find_element_by_xpath(
"//*[@id='blog-posts']/div/ul/li[2]/a")
except NoSuchElementException:
next_page_button = False
else:
driver.close()
break
def tab_restart_browser(webdriver: WebDriver) -> None:
"""
kills the current tab and creates a new one to stop traffic
"""
# note: this technically uses windows, not tabs, due to problems with
# chrome-targeted keyboard commands in Selenium 3 (intermittent
# nonsense WebDriverExceptions are thrown). windows can be reliably
# created, although we do have to detour into JS to do it.
close_other_windows(webdriver)
if webdriver.current_url.lower() == 'about:blank':
return
# Create a new window. Note that it is not practical to use
# noopener here, as we would then be forced to specify a bunch of
# other "features" that we don't know whether they are on or off.
# Closing the old window will kill the opener anyway.
webdriver.execute_script("window.open('')")
# This closes the _old_ window, and does _not_ switch to the new one.
webdriver.close()
# The only remaining window handle will be for the new window;
# switch to it.
# assert len(webdriver.window_handles) == 1
webdriver.switch_to.window(webdriver.window_handles[0])
def close_tab(window_name: str, web_driver: WebDriver):
while True:
try:
web_driver.close()
web_driver.switch_to.window(window_name)
return
except TimeoutException:
continue
def await_session_commands(driver: WebDriver, deps, state_version):
try:
while True:
msg = receive()
if not msg:
raise Exception(
"No more messages from Specstrom, expected RequestAction or End."
)
elif isinstance(msg, RequestAction):
if msg.version == state_version.value:
self.log.info(
f"Performing action in state {state_version.value}: {printer.pretty_print_action(msg.action)}"
)
perform_action(driver, msg.action)
if msg.action.timeout is not None:
self.log.debug(
"Installing change observer")
scripts.install_event_listener(
driver, deps)
state = scripts.query_state(driver, deps)
screenshot(driver, dict_hash(state))
state_version.increment()
send(Performed(state=state))
if msg.action.timeout is not None:
await_events(driver, deps, state_version,
msg.action.timeout)
else:
self.log.warn(
f"Got stale message ({msg}) in state {state_version.value}"
)
send(Stale())
elif isinstance(msg, AwaitEvents):
if msg.version == state_version.value:
self.log.info(
f"Awaiting events in state {state_version.value} with timeout {msg.await_timeout}"
)
scripts.install_event_listener(driver, deps)
await_events(driver, deps, state_version,
msg.await_timeout)
else:
self.log.warn(
f"Got stale message ({msg}) in state {state_version.value}"
)
send(Stale())
elif isinstance(msg, End):
self.log.info("Ending session")
return
else:
raise Exception(f"Unexpected message: {msg}")
finally:
driver.close()
def close_all_tabs_except_given(driver: WebDriver, window_handle: str):
"""
Closes all tabs except the given window handle which can be obtained by driver.current_window_handle
"""
for handle in driver.window_handles:
if handle != window_handle:
driver.switch_to.window(handle)
driver.close()
driver.switch_to.window(window_handle)
def close_all_except_containing_urls(driver: WebDriver, urls: list):
current_window_handle = driver.current_window_handle
for handle in driver.window_handles:
if handle != current_window_handle:
driver.switch_to.window(handle)
window_url = driver.current_url
if not any([url in window_url for url in urls]):
driver.close()
def get_json(browser: WebDriver, context: WebElement, url: str) -> object:
browser.execute_script('arguments[0].href = arguments[1]', context, url)
browser.execute_script('arguments[0].click()', context)
browser.switch_to.window(browser.window_handles[1])
pre = WebDriverWait(browser, timeout=5).until(
EC.presence_of_element_located((By.TAG_NAME, 'pre')))
text = pre.text
obj = json.loads(text)
browser.close()
browser.switch_to.window(browser.window_handles[0])
return obj
def close_other_windows(webdriver: WebDriver) -> None:
"""
close all open pop-up windows and tabs other than the current one
"""
main_handle = webdriver.current_window_handle
windows = webdriver.window_handles
if len(windows) > 1:
for window in windows:
if window != main_handle:
webdriver.switch_to.window(window)
webdriver.close()
webdriver.switch_to.window(main_handle)
def check_selenium_service(command_executor):
browser = WebDriver(command_executor=command_executor,
desired_capabilities=DesiredCapabilities.FIREFOX)
try:
browser.get('https://www.youtube.com/watch?v=dfTWTC7UtcQ')
browser.execute_script("window.scrollTo(0, document.body.offsetHeight);")
WebDriverWait(browser, 5).until(
EC.presence_of_element_located((By.CLASS_NAME, "comment-renderer-text-content"))
)
assert u'comment-renderer-text-content' in browser.page_source
logging.warning("check remote web driver (%s) successful." % command_executor)
finally:
browser.close()
browser.quit()
def check_selenium_service(command_executor):
browser = WebDriver(command_executor=command_executor,
desired_capabilities=DesiredCapabilities.FIREFOX)
try:
browser.get('https://www.youtube.com/watch?v=dfTWTC7UtcQ')
browser.execute_script(
"window.scrollTo(0, document.body.offsetHeight);")
WebDriverWait(browser, 5).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "comment-renderer-text-content")))
assert u'comment-renderer-text-content' in browser.page_source
logging.warning("check remote web driver (%s) successful." %
command_executor)
finally:
browser.close()
browser.quit()
def search_page(url: str,
s: requests.Session = None,
driver: RemoteWebDriver = None) -> tuple:
text = ""
if s is not None:
r = s.get(url)
text = r.text
elif driver is not None:
driver.get(url)
sleep(0.5)
text = driver.page_source
else:
driver = webdriver.Chrome()
driver.get(url)
sleep(0.5)
text = driver.page_source
driver.close()
soup = BeautifulSoup(text, 'lxml')
method = soup.find('span', class_='method-area').text.lower()
url = soup.find('p', class_='url').text
res = get_response(soup)
params = get_parameter(soup)
return (method, url, params, res)
import os
from selenium import webdriver
from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
import time
chrome_options = Options()
chrome_options.add_argument("--headless")
base_url = "http://www.baidu.com/"
# 对应的chromedriver的放置目录
# driver = webdriver.Chrome(executable_path=('/Applications/Google\ Chrome.app/Contents/MacOS/chromedriver'), chrome_options=chrome_options)
driver = RemoteWebDriver("http://127.0.0.1:9515", options=chrome_options)
driver.get(base_url + "/")
start_time = time.time()
print('this is start_time ', start_time)
driver.find_element_by_id("kw").send_keys("selenium webdriver")
driver.find_element_by_id("su").click()
driver.save_screenshot('screen.png')
driver.close()
end_time = time.time()
print('this is end_time ', end_time)
class Browser:
name = None
capname = None
supports_alert = True
def __init__(self):
self.driver = None
self.profiledir = None
self.initialized = False
def initialSetup(self):
pass
def start(self):
if self.driver:
raise RuntimeError('%s is already running' % self.name)
caps = {}
caps.update(getattr(DesiredCapabilities, self.capname))
self.prepare_caps(caps)
self.driver = WebDriver('http://%s:%d/wd/hub' %
(self.args.server_host, self.args.server_port),
desired_capabilities=caps)
self.driver.implicitly_wait(2)
time.sleep(3)
for h in self.driver.window_handles[1:]:
self.driver.switch_to.window(h)
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
self.driver.switch_to_default_content()
if not self.initialized:
self.initialSetup()
self.initialized = True
def quit(self):
try:
if self.driver:
self.driver.quit()
self.driver = None
except http_client.BadStatusLine:
pass
def create_profile(self):
self.profiledir = os.path.join(self.testdir, 'profile', self.name)
if os.path.exists(self.profiledir):
shutil.rmtree(self.profiledir)
os.makedirs(self.profiledir)
return self.profiledir
def set_window_size(self, width, height):
self.driver.set_window_size(width, height)
@property
def url(self):
return self.driver.current_url
def wait_page_load(self):
time.sleep(1)
self.wait_until(lambda d: self.js(
'return !!(window.document&&window.document.readyState==="complete")'
))
def open(self, url):
self.driver.get(url)
self.wait_page_load()
def reload(self):
self.open(self.url)
def wait_until(self, callback):
wait = WebDriverWait(self.driver, 20)
return wait.until(callback)
def ac(self):
return ActionChains(self.driver)
def q(self, selector, context=None):
if context is None:
return self.driver.find_element_by_css_selector(selector)
return context.find_element_by_css_selector(selector)
def qa(self, selector, context=None):
if context is None:
return self.driver.find_elements_by_css_selector(selector)
return context.find_elements_by_css_selector(selector)
def x(self, xpath):
return self.driver.find_element_by_xpath(xpath)
def xa(self, xpath):
return self.driver.find_elements_by_xpath(xpath)
def alert_accept(self):
if self.supports_alert:
Alert(self.driver).accept()
else:
raise RuntimeError('%s not supports alert handling' % self.name)
def js(self, script, *args):
return self.driver.execute_script(script, *args)
def geom(self, element):
return self.geom2(element)[:4]
def geom2(self, element):
return tuple(
map(
round,
self.js(
'''
var elem = arguments[0];
var rect = elem.getBoundingClientRect();
return [rect.left, rect.top, rect.width, rect.height,
rect.left + rect.width, rect.top + rect.height];
''', element)))
def screen_size(self):
sw, sh = self.js('''
return [document.documentElement.clientWidth,
document.documentElement.clientHeight];
''')
return sw, sh
def set_cookie(self, name, value, domain, path):
expires = ''
if value is None:
value = ''
expires = '; expires=Thu, 01 Jan 1970 00:00:00 GMT'
cookie = '%s=%s; domain=%s; path=%s%s' % (name, value, domain, path,
expires)
self.js('document.cookie=%s' % json.dumps(cookie))
def screenshot(self, elem=None):
ss = self.driver.get_screenshot_as_png()
io = util.BytesIO(ss)
img = Image.open(io)
if elem is not None:
x, y, w, h = tuple(map(int, self.geom(elem)))
img = img.crop((x, y, x + w, y + h))
return img.convert('RGB')
def download(self, url, filename):
dlpath = os.path.join(self.browserdir, filename)
if not os.path.exists(dlpath) or os.stat(
dlpath).st_mtime < time.time() - 60 * 60 * 24:
util.download(url, dlpath)
return dlpath
def read_file_as_base64(self, filename):
fp = open(filename, 'rb')
try:
data = fp.read()
finally:
fp.close()
data = base64.b64encode(data)
try:
data = str(data, 'ascii')
except TypeError:
pass
return data
def cerrar_sesion(webdriver: WebDriver, result_list: ValidacionResultList, correo: Correo):
timeout_cierre_sesion = 10
resultado_cierre_sesion = ResultStep()
resultado_cierre_sesion.inicializar_tiempo_de_ejecucion()
cierre_sesion_exitosa = False
# verifica si se tiene error de credenciales, por lo cual si se tiene este error, se establece el mensaje
# de error y envia el result como finalizado, esto debido a que no puede localizar el boton de cierre de
# sesion sin antes haberse loggeado dentro de la plataforma
if result_list.result_validacion_acceso_portal_owa.error_inicio_de_sesion_credenciales_erroneas:
resultado_cierre_sesion.finalizar_tiempo_de_ejecucion()
resultado_cierre_sesion.establecer_tiempo_de_ejecucion()
resultado_cierre_sesion.validacion_correcta = False
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions.\
CERRAR_SESION_MSG_ERROR_CREDENCIALES_OWA.format(result_list.result_validacion_acceso_portal_owa.
msg_error_de_credenciales)
result_list.result_validacion_cierre_sesion = resultado_cierre_sesion
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_MSG_ERROR_CREDENCIALES_OWA.format(
result_list.result_validacion_acceso_portal_owa.msg_error_de_credenciales))
webdriver.close()
webdriver.quit()
return result_list
# verifica si hay error en plataforma, en caso de ser asi, intenta realizar n intentos para volver a loggearse
# y verificar si ingreso correctamente al buzon de entrada para navegar entre las carpetas
if ValidacionesHTML.verificar_error_plataforma(webdriver):
resultado_cierre_sesion = ValidacionesHTML.intento_ingreso_nuevamente_al_portal(
resultado_cierre_sesion, correo, webdriver, step_evaluacion='cierre de sesion')
try:
webdriver.refresh()
time.sleep(5)
# verifica que no haya algun dialogo que impida el cierre de sesion
ValidacionesHTML.verificar_dialogo_de_interrupcion(
webdriver, resultado_cierre_sesion)
# intenta salir de la sesion ejecutando un script js el cual simula un clic en el boton de cierre de sesion
if AccionesHtml.owa_descubierto == 2010:
time.sleep(4)
elemento_html_btn_cerrar_sesion = webdriver.find_element_by_id(
constantes_webdriver_actions.CERRAR_SESION_CIERRE_SESION_ID_BTN_CIERRE_SESION_OWA_2010)
time.sleep(4)
elemento_html_btn_cerrar_sesion.click()
time.sleep(4)
elif AccionesHtml.owa_descubierto == 2016:
time.sleep(4)
boton_cierre_sesion_owa_2016 = webdriver.execute_script(
constantes_webdriver_actions.JS_LOCATE_DIV_CONTENT_BTN_CIERRE_SESION_OWA_2016)
time.sleep(4)
boton_cierre_sesion_owa_2016.click()
time.sleep(8)
boton_cierre_sesion_owa_2016 = webdriver.execute_script(
constantes_webdriver_actions.JS_LOCATE_BTN_CIERRE_SESION_OWA_2016)
boton_cierre_sesion_owa_2016.click()
elif AccionesHtml.owa_descubierto == 2013:
boton_cierre_sesion_owa_2013 = webdriver.execute_script(
constantes_webdriver_actions.JS_LOCATE_DIV_CONTENT_BTN_CIERRE_SESION_OWA_2013)
boton_cierre_sesion_owa_2013.click()
time.sleep(8)
boton_cierre_sesion_owa_2013 = webdriver.execute_script(
constantes_webdriver_actions.JS_LOCATE_BTN_CIERRE_SESION_OWA_2013_SPANISH)
if boton_cierre_sesion_owa_2013 is None:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_BTN_CIERRE_SIGN_OUT)
boton_cierre_sesion_owa_2013 = webdriver.execute_script(
constantes_webdriver_actions.JS_LOCATE_BTN_CIERRE_SESION_OWA_2013_ENGLISH)
boton_cierre_sesion_owa_2013.click()
# obtiene la url actual como una cadena
time.sleep(2)
AccionesHtml.log.info(constantes_webdriver_actions.CERRAR_SESION_LOG_INFO_CIERRE_SESION_TITLE_URL
.format(webdriver.title, webdriver.current_url))
webdriver.refresh()
time.sleep(2)
# verifica que nos encontremos en la pagina de cierre de sesion del OWA verifica que el title de la pagina
# contenfa Outlook
condicion_contenido_en_title = EC.title_contains(constantes_webdriver_actions.
CERRAR_SESION_TITLE_CIERRE_SESION)
WebDriverWait(webdriver, timeout_cierre_sesion).until(condicion_contenido_en_title)
cierre_sesion_exitosa = True
except selExcep.NoSuchElementException as e:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_NO_SUCH_ELEM_EXCEP.format(
FormatUtils.formatear_excepcion(e)))
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions. \
CERRAR_SESION_LOG_ERROR_NO_SUCH_ELEM_EXCEP.format(FormatUtils.formatear_excepcion(e))
resultado_cierre_sesion.validacion_correcta = False
except selExcep.ElementClickInterceptedException as e:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_ELEM_INTERCEP_EXCEP.format(
FormatUtils.formatear_excepcion(e)))
webdriver.refresh()
time.sleep(2)
AccionesHtml.cerrar_sesion(webdriver, result_list, correo)
except selExcep.TimeoutException as e:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_TIMEOUT_EXCEP.format(
FormatUtils.formatear_excepcion(e)))
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions. \
CERRAR_SESION_LOG_ERROR_TIMEOUT_EXCEP.format(FormatUtils.formatear_excepcion(e))
resultado_cierre_sesion.validacion_correcta = False
except selExcep.WebDriverException as e:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_WEBDRIVER_EXCEP
.format(FormatUtils.formatear_excepcion(e)))
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions. \
CERRAR_SESION_LOG_ERROR_WEBDRIVER_EXCEP.format(FormatUtils.formatear_excepcion(e))
resultado_cierre_sesion.validacion_correcta = False
except AttributeError:
AccionesHtml.log.error(constantes_webdriver_actions.CERRAR_SESION_LOG_ERROR_ATRIBUTE_ERROR_EXCEP)
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions. \
CERRAR_SESION_LOG_ERROR_ATRIBUTE_ERROR_EXCEP
resultado_cierre_sesion.validacion_correcta = False
finally:
# verifica que no haya algun mensaje de error en la plataforma, en caso contrario se muestra el mensaje de
# error que aparace en la plataforma dentro del result
if ValidacionesHTML.verificar_error_plataforma(webdriver):
resultado_cierre_sesion.validacion_correcta = False
msg_error = ValidacionesHTML.obtener_mensaje_error_plataforma(webdriver)
resultado_cierre_sesion.mensaje_error = constantes_webdriver_actions.CERRAR_SESION_ERROR_PLATAFORMA. \
format(msg_error)
cierre_sesion_exitosa = False
AccionesHtml.log.info(constantes_webdriver_actions.CERRAR_SESION_LOG_INFO_CIERRE_WEB_DRIVER)
webdriver.close()
webdriver.quit()
if cierre_sesion_exitosa:
AccionesHtml.log.info(constantes_webdriver_actions.CERRAR_SESION_LOG_INFO_CIERRE_SESION_EXITOSA)
resultado_cierre_sesion.mensaje_error = constantes_json.OUTPUT_EXITOSO_3_1
resultado_cierre_sesion.validacion_correcta = True
resultado_cierre_sesion.finalizar_tiempo_de_ejecucion()
resultado_cierre_sesion.establecer_tiempo_de_ejecucion()
result_list.result_validacion_cierre_sesion = resultado_cierre_sesion
return result_list
class Browser:
name = None
capname = None
supports_alert = True
def __init__(self):
self.driver = None
self.profiledir = None
self.initialized = False
def initialSetup(self):
pass
def start(self):
if self.driver:
raise RuntimeError('%s is already running' % self.name)
caps = {}
caps.update(getattr(DesiredCapabilities, self.capname))
self.prepare_caps(caps)
self.driver = WebDriver(
'http://%s:%d/wd/hub' % (self.args.server_host, self.args.server_port),
desired_capabilities = caps
)
self.driver.implicitly_wait(2)
time.sleep(3)
for h in self.driver.window_handles[1:]:
self.driver.switch_to.window(h)
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
self.driver.switch_to_default_content()
if not self.initialized:
self.initialSetup()
self.initialized = True
def quit(self):
try:
if self.driver:
self.driver.quit()
self.driver = None
except http_client.BadStatusLine:
pass
def create_profile(self):
self.profiledir = os.path.join(self.testdir, 'profile', self.name)
if os.path.exists(self.profiledir):
shutil.rmtree(self.profiledir)
os.makedirs(self.profiledir)
return self.profiledir
def set_window_size(self, width, height):
self.driver.set_window_size(width, height)
@property
def url(self):
return self.driver.current_url
def wait_page_load(self):
time.sleep(1)
self.wait_until(lambda d: self.js('return !!(window.document&&window.document.readyState==="complete")'))
def open(self, url):
self.driver.get(url)
self.wait_page_load()
def reload(self):
self.open(self.url)
def wait_until(self, callback):
wait = WebDriverWait(self.driver, 20)
return wait.until(callback)
def ac(self):
return ActionChains(self.driver)
def q(self, selector, context = None):
if context is None:
return self.driver.find_element_by_css_selector(selector)
return context.find_element_by_css_selector(selector)
def qa(self, selector, context = None):
if context is None:
return self.driver.find_elements_by_css_selector(selector)
return context.find_elements_by_css_selector(selector)
def x(self, xpath):
return self.driver.find_element_by_xpath(xpath)
def xa(self, xpath):
return self.driver.find_elements_by_xpath(xpath)
def alert_accept(self):
if self.supports_alert:
Alert(self.driver).accept()
else:
raise RuntimeError('%s not supports alert handling' % self.name)
def js(self, script, *args):
return self.driver.execute_script(script, *args)
def geom(self, element):
return self.geom2(element)[:4]
def geom2(self, element):
return tuple(map(round, self.js('''
var elem = arguments[0];
var rect = elem.getBoundingClientRect();
return [rect.left, rect.top, rect.width, rect.height,
rect.left + rect.width, rect.top + rect.height];
''', element)))
def screen_size(self):
sw, sh = self.js('''
return [document.documentElement.clientWidth,
document.documentElement.clientHeight];
''')
return sw, sh
def set_cookie(self, name, value, domain, path):
expires = ''
if value is None:
value = ''
expires = '; expires=Thu, 01 Jan 1970 00:00:00 GMT'
cookie = '%s=%s; domain=%s; path=%s%s' % (name, value, domain, path, expires)
self.js('document.cookie=%s' % json.dumps(cookie))
def screenshot(self, elem = None):
ss = self.driver.get_screenshot_as_png()
io = util.BytesIO(ss)
img = Image.open(io)
if elem is not None:
x, y, w, h = tuple(map(int, self.geom(elem)))
img = img.crop((x, y, x + w, y + h))
return img.convert('RGB')
def download(self, url, filename):
dlpath = os.path.join(self.browserdir, filename)
if not os.path.exists(dlpath) or os.stat(dlpath).st_mtime < time.time() - 60 * 60 * 24:
util.download(url, dlpath)
return dlpath
def read_file_as_base64(self, filename):
fp = open(filename, 'rb')
try:
data = fp.read()
finally:
fp.close()
data = base64.b64encode(data)
try:
data = str(data, 'ascii')
except TypeError:
pass
return data
def start_scrapp(driver: WebDriver,
loginEvent=None,
loadedEvent=None,
queue=None):
# loginEvent.wait()
#TODO criar o Scrapper
print(100 * "#")
#Faça um scroll na página até alcançar todos os filmes
#TODO otimizar esse looping
movies_sources = []
for i, so in enumerate(['az', 'za']):
driver.get(config.MAIN_URL + '/browse/genre/34399?so=' + so)
previous_html = ""
current_html = driver.page_source
print('STATUS: Finding all movies on netflix...[' + str(i + 1) +
'/2] ',
end='',
flush=True)
while current_html != previous_html:
previous_html = current_html
driver.execute_script(
"window.scrollTo(0,document.body.scrollHeight);")
sleep(1.2) # Pode variar de acordo com a velocidade da internet
current_html = driver.page_source
break
print('OK')
print('STATUS: Saving innerHTML of all movies...[' + str(i + 1) +
'/2] ',
end='',
flush=True)
movies_sources += list(
map(lambda p: bs(p.get_attribute('innerHTML'), 'html.parser'),
driver.find_elements_by_class_name('slider-item')))
print('OK')
break
all_sources = sorted(movies_sources,
key=lambda s: s.find('a')['aria-label'])
aux_set = set()
movies_sources = []
for s in all_sources:
if s not in aux_set:
aux_set.add(s)
movies_sources.append(s)
del aux_set
print('STATUS: Starting to get information... ')
if not os.path.exists(config.FOLDER_NAME):
os.mkdir(config.FOLDER_NAME)
driver.close()
for i, s_movie in enumerate(movies_sources):
t = Thread(target=scrapp_image,
args=(s_movie, ),
name='movie_' + str(i))
t.start()
# MAX_THREADS + gui thread + main thread (prompt)
while active_count() > config.MAX_THREADS:
sleep(.5)
def close_current_tab(driver: WebDriver):
"""Close current tab and switch to first tab"""
tabs = driver.window_handles
driver.close()
driver.switch_to.window(tabs[0])
