def scrape_league_quotes(brow: webdriver, league_name: str) -> webdriver:
if not brow:
brow = open_browser()
brow.get(utl.get_league_url(league_name))
brow.refresh()
time.sleep(5)
for i in range(cfg.MATCHES_TO_SCRAPE):
matches = find_all_matches(brow=brow, league_name=league_name)
# Select the match or continue to the next league if done
try:
match = matches[i]
except IndexError:
break
match_dt = extract_match_datetime(brow, match)
if utl.match_is_out_of_range(match_dt):
break
else:
match.click()
# Fill "matches" table in the db
last_id = insert_match(brow=brow,
league_name=league_name,
match_dt=match_dt)
# Fill "quotes" table in the db
insert_quotes(brow, last_id)
return_to_league_page(brow=brow)
return brow
def buy(driver: webdriver):
url = 'https://market.m.taobao.com/app/sj/shop-membership-center/pages/index?spm=a1z10.4-b-s.w5003-22059585455.1.7f86274aL8QKmA' \
'&wh_weex=true&wx_navbar_transparent=true&sellerId=2360209412&scene=taobao_shop'
btn_buy = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//div[@class='gift-act-btn']"
btn_word = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//span[@class='gift-act-btn-text']"
btn = "//div[@class='btnWarp']"
word = "//span[@class='btn']"
count = 0
login_time = "2020-06-10 13:23:00"
buy_time = "2020-06-10 13:23:55"
isFirst = True
wait = WebDriverWait(driver, 10, 0.5)
while True:
if datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') < buy_time:
# 提前登录
if isFirst and datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f') >= login_time:
driver.maximize_window()
driver.get(url)
driver.implicitly_wait(2)
login(driver)
isFirst = False
continue
else:
# 如果连续抢60次都抢不到 那么就放弃吧
count += 1
if count > 60:
break
print("即将开始抢购...")
print(datetime.datetime.now())
# 重新进页面
driver.refresh()
# 开始页面的唇膏抢购按钮
buy_btn = wait.until(
lambda driver: driver.find_element_by_xpath(btn_buy))
# 开始页面的唇膏抢购按钮上的文字
word_btn = wait.until(
lambda driver: driver.find_element_by_xpath(btn_word))
if word_btn.text == '立即兑换':
# 如果抢购按钮上的文字为立即兑换 则点击按钮跳转详情页
buy_btn.click()
# 详情页中的购买按钮
targetBtn = wait.until(
lambda driver: driver.find_element_by_xpath(btn))
# 详情页中的购买按钮上的文字
targetBtnWord = wait.until(
lambda driver: driver.find_element_by_xpath(word))
if targetBtnWord.text != "已抢光":
# 如果详情页中购买按钮上的文字不为已枪光 则点击购买
targetBtn.click()
time.sleep(5)
# 抢到了就退出...
break
print("抢购结束...")
def borrar_archivo_claro_drive(webdriver_test_ux: webdriver, jsonEval, nombre_archivo_sin_ext: str, ext_archivo: str):
tiempo_step_inicio = Temporizador.obtener_tiempo_timer()
fecha_inicio = Temporizador.obtener_fecha_tiempo_actual()
try:
webdriver_test_ux.refresh()
archivo_por_eliminar = WebDriverWait(webdriver_test_ux, 20).until(
EC.element_to_be_clickable(
(By.XPATH, '//span[@class="name-without-extension"][text()="{} "]'.format(nombre_archivo_sin_ext))))
archivo_por_eliminar.click()
btn_borrar = WebDriverWait(webdriver_test_ux, 20).until(EC.element_to_be_clickable(
(By.XPATH, '//input[@type="button"][@class="menuItem svg deleteImage icon-delete icon-32"]')))
btn_borrar.click()
jsonEval["steps"][3]["output"][0]["status"] = jsonConst.SUCCESS
jsonEval["steps"][3]["status"] = jsonConst.SUCCESS
jsonEval["steps"][3]["output"][0]["output"] = 'Se realiza el borrado del archivo correctamente'
except NoSuchElementException as e:
jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["output"][0][
"output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg)
except ElementClickInterceptedException as e:
jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["output"][0][
"output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg)
except TimeoutException as e:
jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["status"] = jsonConst.FAILED
jsonEval["steps"][3]["output"][0][
"output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg)
tiempo_step_final = Temporizador.obtener_tiempo_timer() - tiempo_step_inicio
fecha_fin = Temporizador.obtener_fecha_tiempo_actual()
jsonEval["steps"][3]["time"] = FormatUtils.truncar_float_cadena(tiempo_step_final)
jsonEval["steps"][3]["start"] = fecha_inicio
jsonEval["steps"][3]["end"] = fecha_fin
return jsonEval
def cerrar_sesion_claro_drive(webdriver_test_ux: webdriver, jsonEval):
tiempo_step_inicio = Temporizador.obtener_tiempo_timer()
fecha_inicio = Temporizador.obtener_fecha_tiempo_actual()
try:
webdriver_test_ux.refresh()
boton_ajustes = WebDriverWait(webdriver_test_ux, 10).until(EC.element_to_be_clickable((By.ID, 'expand')))
boton_ajustes.click()
boton_cerrar_sesion = WebDriverWait(webdriver_test_ux, 20).until(
EC.element_to_be_clickable((By.XPATH, '//li[@data-id="logout"]/a')))
boton_cerrar_sesion.click()
WebDriverWait(webdriver_test_ux, 10).until(EC.presence_of_element_located((By.ID, 'login')))
jsonEval["steps"][4]["output"][0]["status"] = jsonConst.SUCCESS
jsonEval["steps"][4]["status"] = jsonConst.SUCCESS
jsonEval["steps"][4]["output"][0]["output"] = 'Se cierra sesion correctamente'
except NoSuchElementException as e:
jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg)
except ElementClickInterceptedException as e:
jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg)
except TimeoutException as e:
jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg)
except ElementNotInteractableException as e:
jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["status"] = jsonConst.FAILED
jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg)
tiempo_step_final = Temporizador.obtener_tiempo_timer() - tiempo_step_inicio
fecha_fin = Temporizador.obtener_fecha_tiempo_actual()
jsonEval["steps"][4]["time"] = FormatUtils.truncar_float_cadena(tiempo_step_final)
jsonEval["steps"][4]["start"] = fecha_inicio
jsonEval["steps"][4]["end"] = fecha_fin
return jsonEval
def __book_slots(driver: webdriver, slots: list) -> bool:
for slot in slots:
# Attempt to book slot
logger.info(f"Attempting to reserve time slot...")
driver.execute_script(slot)
time.sleep(1)
success_alert = driver.find_element_by_id("alertBookingSuccess")
if success_alert.get_attribute("style") == "display: block;":
# Booking successful
logger.info(f"Booked time slot!")
return True
else:
logger.info(f"Failed to book time slot...")
driver.refresh()
logger.info(f"None of the preferred time slots were available")
return False
def solicitar_cita(driver: webdriver, context: CustomerProfile):
driver.execute_script("enviar('solicitud');")
for i in range(REFRESH_PAGE_CYCLES):
resp_text = body_text(driver)
if "Por favor, valide el Captcha para poder continuar" in resp_text:
success = process_captcha(driver, context, partially=True)
if not success:
return None
return solicitar_cita(driver, context)
elif "Seleccione la oficina donde solicitar la cita" in resp_text:
logging.info("Towns hit! :)")
# Office selection:
time.sleep(0.3)
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.ID, "btnSiguiente"))
)
except TimeoutException:
logging.error("Timed out waiting for offices to load")
return None
res = select_office(driver, context)
if res is None:
time.sleep(5)
driver.refresh()
continue
btn = driver.find_element_by_id("btnSiguiente")
btn.send_keys(Keys.ENTER)
return True
elif "En este momento no hay citas disponibles" in resp_text:
time.sleep(5)
driver.refresh()
continue
else:
logging.info("No towns")
return None
def auto_alert_accept(driver: webdriver):
"""Checking js-alert and accpet
This method wrapped by try-catch block.
Except: NoAlertPresentException
return values: No exception = True, Exception occured = False
"""
try:
result = driver.switch_to_alert()
result.accept()
return True
except NoAlertPresentException as e:
""" There was no js-alert"""
print("There is no js-alert")
print(e.__class__.__name__)
print(e)
driver.refresh()
return False
except Exception as e:
""" Unexpected exception"""
print("Unexpected except")
assert e.__class__.__name__ == 'NameError'
return False
def get_car_data(self, driver: webdriver, i: int) -> None:
c = inspect.currentframe()
try:
WebDriverWait(driver, 5).until(
EC.element_to_be_clickable(
(By.XPATH,
'/html/body/div[1]/div[4]/div[2]/div[5]/table/tbody/tr[' +
str(i) + ']/td[5]/p/a[@class="a_list"]'))).click()
except Exception as e:
print('Can\'t click on car %d. Reason %s' % (i, e))
driver.refresh()
if self.alert_present(driver):
return False
tabs = driver.window_handles
try:
driver.switch_to.window(tabs[1])
except Exception as e:
print('Can\'t switch to new window. Reason %s' % e)
self.download_images(self.get_img(driver))
car: Dict[str] = {}
car['category'] = self.category
car['category_url'] = self.category_url
try:
car['title'] = self.ko_translate(
driver.find_element_by_xpath(
'/html/body/div[1]/div[1]/h2').text, "en")
car['title-seo'] = self.ko_translate(
driver.find_element_by_xpath(
'/html/body/div[1]/div[1]/h2').text, "en")
except:
car['title'] = ''
driver.refresh()
try:
car['price'] = int(
driver.find_element_by_xpath(
'/html/body/div[1]/div[2]/div[2]/p/strong/em').text.
replace(',', '')) * 9.10
except:
car['price'] = ''
driver.refresh()
try:
car['description'] = '<div class="paper-tit"><font style="vertical-align: inherit;"><font style="vertical-align: inherit;">Протокол осмотра / осмотра автомобилей, выставленных на аукцион</font></font></div>'
el = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable((
By.CSS_SELECTOR,
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table'
)))
if (len(el.get_attribute("outerHTML")) > 5000):
i = 0
while (i <= int(len(el.get_attribute("outerHTML")) / 4999)):
if (i == int(len(el.get_attribute("outerHTML")) / 4999)):
car['description'] += self.rm_new_line(
self.ko_translate(
el.get_attribute("outerHTML")
[i * 4999:len(el.get_attribute("outerHTML"))],
"ru"))
else:
car['description'] += self.rm_new_line(
self.ko_translate(
el.get_attribute("outerHTML")[i * 4999:i +
1 * 4999], "ru"))
i += 1
else:
car['description'] += self.rm_new_line(
self.ko_translate(el.get_attribute("outerHTML"), "ru"))
except Exception as e:
car['description'] += ''
print(c.f_lineno)
print('Can\'t get protocol view. Reason %s' % e)
driver.refresh()
try:
car['description'] += '<h2 class="page-subtit mt60"><font style="vertical-align: inherit"><font style="vertical-align: inherit">Детали автомобиля</font></font></h2>'
el = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable((
By.CSS_SELECTOR,
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div.vehicle-detail > div.vehicle-detail_bar > table.tbl-v02'
)))
car['description'] += self.rm_new_line(
self.ko_translate(el.get_attribute("outerHTML"), "ru"))
except Exception as e:
car['description'] += ''
print(c.f_lineno)
print('Can\'t get car details. Reason %s' % e)
driver.refresh()
try:
car['description'] += '<h2 class="page-subtit mt60" id="view-status"><font style="vertical-align: inherit"><font style="vertical-align: inherit">Состояние кузова автомобиля</font></font></h2>'
el = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable((
By.CSS_SELECTOR,
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div.tab-status'
)))
car['description'] += self.rm_new_line(
self.ko_translate(el.get_attribute("outerHTML"), "ru"))
except Exception as e:
car['description'] += ''
print(c.f_lineno)
print('Can\'t get car condition. Reason %s' % e)
driver.refresh()
try:
car['description'] += '<h2 class="page-subtit mt60"><font style="vertical-align: inherit;"><font style="vertical-align: inherit;">Видео автомобиля</font></font></h2>'
el = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable((
By.CSS_SELECTOR,
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div:nth-child(5)'
)))
car['description'] += self.rm_new_line(
self.ko_translate(el.get_attribute("outerHTML"), "ru"))
except Exception as e:
car['description'] += ''
print(c.f_lineno)
print('Can\'t get car control list. Reason %s' % e)
driver.refresh()
try:
year = int(
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table > tbody > tr:nth-child(5) > td:nth-child(4)'
).text)
except Exception as e:
print('Can\'t get car year. Reason %s' % e)
driver.refresh()
try:
mark = self.ko_translate(
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table > tbody > tr:nth-child(4) > td:nth-child(5)'
).text, "en")
except Exception as e:
print('Can\'t get car mark. Reason %s' % e)
driver.refresh()
try:
color = self.ko_translate(
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(3) > td:nth-child(4)'
).text, "ru")
except Exception as e:
print('Can\'t get car color. Reason %s' % e)
driver.refresh()
try:
fulel_data = {
"가솔린": "Бензин",
"디젤": "Дизель",
"LPG": "LPG",
"LPI하이브리드": "LPG гибрид",
"가솔린하이브리드": "Бензиновый гибрид",
"디젤하이브리드": "Дизельный гибрид",
"전기": "Электрокар",
"가솔린/LPG": "Бензин/LPG"
}
fuel = fulel_data[driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(4) > td:nth-child(4)'
).text]
except Exception as e:
fuel = "Дизель"
print('Can\'t get car fuel. Reason %s' % e)
driver.refresh()
try:
res = re.findall(
"\d+",
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(5) > td:nth-child(4)'
).text)
displacement = int(''.join(res))
except Exception as e:
print('Can\'t get car displacement. Reason %s' % e)
driver.refresh()
try:
transmission = driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(2) > td:nth-child(4)'
).text == "자동" if "Автомат" else "Механика"
except Exception as e:
print('Can\'t get car transmission. Reason %s' % e)
driver.refresh()
try:
car_type = self.ko_translate(
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(6) > td:nth-child(2)'
).text, "ru")
except Exception as e:
print('Can\'t get car type. Reason %s' % e)
driver.refresh()
try:
lot_number = self.ko_translate(
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-tit > p > strong'
).text, "ru")
except Exception as e:
print('Can\'t get car type. Reason %s' % e)
driver.refresh()
try:
r = re.findall(
"\d+",
driver.find_element_by_css_selector(
'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(1) > td:nth-child(4)'
).text)
distance_driven = int(''.join(r))
except Exception as e:
print('Can\'t get car distance driven. Reason %s' % e)
driver.refresh()
car['images'] = self.get_img_str(driver)
car['count'] = '0'
car['activation'] = '1'
car['currency'] = 'USD'
car['recomended'] = '0'
car['new'] = '0'
car['weight'] = '0'
car['article'] = lot_number
car['properties'] = (
'Цвет=[type=assortmentCheckBox value=%s product_margin=Желтый|Белый|Серебро|Красный|Фиолетовый|Оранжевый|Зеленый|Серый|Золото|Коричневый|Голубой|Черный|Бежевый]&Кузов=[type=assortmentCheckBox value=%s product_margin=Универсал|Фургон|Фура|Трактор|Седан|Родстер|Пикап|Мотоцикл|Минивен|Хэтчбек|Кроссовер|Купе|Кабриолет|Багги]&Пробег=%d&Двигатель=%d&Год=%d&Трансмиссия=[type=assortmentCheckBox value=%s product_margin=Механика|Автомат]&Топливо=[type=assortmentCheckBox value=%s product_margin=Дизель|Бензин|Газ]&Модель=%s&Марка=%s&Номер лота=%s&Аукцион=lotteautoauction'
% (color, car_type, distance_driven, displacement, year,
transmission, fuel, mark, self.category, lot_number))
try:
driver.close()
driver.switch_to.window(tabs[0])
except Exception as e:
print('Can\'t switch to old window. Reason %s' % e)
self.write_csv(car)
def cycle_cita(driver: webdriver, context: CustomerProfile):
driver.delete_all_cookies()
try:
driver.execute_script("window.localStorage.clear();")
driver.execute_script("window.sessionStorage.clear();")
except Exception as e:
logging.error(e)
pass
if context.fast_forward_url:
while True:
try:
driver.set_page_load_timeout(300 if context.first_load else 50)
driver.get(context.fast_forward_url)
except TimeoutException:
logging.error("Timed out loading initial page")
continue
break
context.first_load = False
session_id = driver.get_cookie("JSESSIONID").get("value")
logging.info(session_id)
else:
driver.get(
"https://sede.administracionespublicas.gob.es/icpplus/index.html")
time.sleep(1) # Let the user actually see something!
# Select "Barcelona"
select = Select(driver.find_element_by_id("form"))
select.select_by_visible_text(context.city)
btn = driver.find_element_by_id("btnAceptar")
btn.send_keys(Keys.ENTER)
# 2. Tramite selection:
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.ID, "tramiteGrupo[1]")))
except TimeoutException:
logging.error("Timed out waiting for tramite to load")
return None
select = Select(driver.find_element_by_id("tramiteGrupo[1]"))
# Select "Huellos"
select.select_by_value(context.operation_code.value)
btn = driver.find_element_by_id("btnAceptar")
btn.send_keys(Keys.ENTER)
# 3. Instructions page:
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.ID, "btnEntrar")))
except TimeoutException:
logging.error("Timed out waiting for Instructions page to load")
return None
btn = driver.find_element_by_id("btnEntrar")
btn.send_keys(Keys.ENTER)
# 4. Data form:
success = False
if context.operation_code == OperationType.TOMA_HUELLAS:
success = toma_huellas_step2(driver, context)
elif context.operation_code == OperationType.RECOGIDA_DE_TARJETA:
success = recogida_de_tarjeta_step2(driver, context)
elif context.operation_code == OperationType.SOLICITUD:
success = solicitud_step2(driver, context)
if not success:
return None
try:
wait_exact_time(driver, context)
except TimeoutException:
logging.error("Timed out waiting for exact time")
return None
# 5. Solicitar cita:
btn = driver.find_element_by_id("btnEnviar")
btn.send_keys(Keys.ENTER)
for i in range(REFRESH_PAGE_CYCLES):
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.TAG_NAME, "body")))
except TimeoutException:
logging.error("Timed out waiting for body to load")
return None
resp_text = driver.find_element_by_tag_name("body").text
if "Seleccione la oficina donde solicitar la cita" in resp_text:
logging.info("Towns hit! :)")
# 6. Office selection:
time.sleep(0.3)
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.ID, "btnSiguiente")))
except TimeoutException:
logging.error("Timed out waiting for offices to load")
return None
res = select_office(driver, context)
if res is None:
time.sleep(5)
driver.refresh()
continue
btn = driver.find_element_by_id("btnSiguiente")
btn.send_keys(Keys.ENTER)
break
elif "En este momento no hay citas disponibles" in resp_text:
time.sleep(5)
driver.refresh()
continue
else:
logging.info("No towns")
return None
# 7. phone-mail:
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.ID, "emailDOS")))
logging.info("Email page hit")
except TimeoutException:
logging.error("Timed out waiting for phone/email to load")
return None
element = driver.find_element_by_id("txtTelefonoCitado")
element.send_keys(context.phone) # phone num
element = driver.find_element_by_id("emailUNO")
element.send_keys(context.email)
element = driver.find_element_by_id("emailDOS")
element.send_keys(context.email)
btn = driver.find_element_by_id("btnSiguiente")
btn.send_keys(Keys.ENTER)
return cita_selection(driver, context)
