def scrape_league_quotes(brow: webdriver, league_name: str) -> webdriver: if not brow: brow = open_browser() brow.get(utl.get_league_url(league_name)) brow.refresh() time.sleep(5) for i in range(cfg.MATCHES_TO_SCRAPE): matches = find_all_matches(brow=brow, league_name=league_name) # Select the match or continue to the next league if done try: match = matches[i] except IndexError: break match_dt = extract_match_datetime(brow, match) if utl.match_is_out_of_range(match_dt): break else: match.click() # Fill "matches" table in the db last_id = insert_match(brow=brow, league_name=league_name, match_dt=match_dt) # Fill "quotes" table in the db insert_quotes(brow, last_id) return_to_league_page(brow=brow) return brow
def buy(driver: webdriver): url = 'https://market.m.taobao.com/app/sj/shop-membership-center/pages/index?spm=a1z10.4-b-s.w5003-22059585455.1.7f86274aL8QKmA' \ '&wh_weex=true&wx_navbar_transparent=true&sellerId=2360209412&scene=taobao_shop' btn_buy = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//div[@class='gift-act-btn']" btn_word = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//span[@class='gift-act-btn-text']" btn = "//div[@class='btnWarp']" word = "//span[@class='btn']" count = 0 login_time = "2020-06-10 13:23:00" buy_time = "2020-06-10 13:23:55" isFirst = True wait = WebDriverWait(driver, 10, 0.5) while True: if datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') < buy_time: # 提前登录 if isFirst and datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S.%f') >= login_time: driver.maximize_window() driver.get(url) driver.implicitly_wait(2) login(driver) isFirst = False continue else: # 如果连续抢60次都抢不到 那么就放弃吧 count += 1 if count > 60: break print("即将开始抢购...") print(datetime.datetime.now()) # 重新进页面 driver.refresh() # 开始页面的唇膏抢购按钮 buy_btn = wait.until( lambda driver: driver.find_element_by_xpath(btn_buy)) # 开始页面的唇膏抢购按钮上的文字 word_btn = wait.until( lambda driver: driver.find_element_by_xpath(btn_word)) if word_btn.text == '立即兑换': # 如果抢购按钮上的文字为立即兑换 则点击按钮跳转详情页 buy_btn.click() # 详情页中的购买按钮 targetBtn = wait.until( lambda driver: driver.find_element_by_xpath(btn)) # 详情页中的购买按钮上的文字 targetBtnWord = wait.until( lambda driver: driver.find_element_by_xpath(word)) if targetBtnWord.text != "已抢光": # 如果详情页中购买按钮上的文字不为已枪光 则点击购买 targetBtn.click() time.sleep(5) # 抢到了就退出... break print("抢购结束...")
def borrar_archivo_claro_drive(webdriver_test_ux: webdriver, jsonEval, nombre_archivo_sin_ext: str, ext_archivo: str): tiempo_step_inicio = Temporizador.obtener_tiempo_timer() fecha_inicio = Temporizador.obtener_fecha_tiempo_actual() try: webdriver_test_ux.refresh() archivo_por_eliminar = WebDriverWait(webdriver_test_ux, 20).until( EC.element_to_be_clickable( (By.XPATH, '//span[@class="name-without-extension"][text()="{} "]'.format(nombre_archivo_sin_ext)))) archivo_por_eliminar.click() btn_borrar = WebDriverWait(webdriver_test_ux, 20).until(EC.element_to_be_clickable( (By.XPATH, '//input[@type="button"][@class="menuItem svg deleteImage icon-delete icon-32"]'))) btn_borrar.click() jsonEval["steps"][3]["output"][0]["status"] = jsonConst.SUCCESS jsonEval["steps"][3]["status"] = jsonConst.SUCCESS jsonEval["steps"][3]["output"][0]["output"] = 'Se realiza el borrado del archivo correctamente' except NoSuchElementException as e: jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][3]["status"] = jsonConst.FAILED jsonEval["steps"][3]["output"][0][ "output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg) except ElementClickInterceptedException as e: jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][3]["status"] = jsonConst.FAILED jsonEval["steps"][3]["output"][0][ "output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg) except TimeoutException as e: jsonEval["steps"][3]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][3]["status"] = jsonConst.FAILED jsonEval["steps"][3]["output"][0][ "output"] = 'No fue posible realizar el borrado del archivo correctamente: {}'.format(e.msg) tiempo_step_final = Temporizador.obtener_tiempo_timer() - tiempo_step_inicio fecha_fin = Temporizador.obtener_fecha_tiempo_actual() jsonEval["steps"][3]["time"] = FormatUtils.truncar_float_cadena(tiempo_step_final) jsonEval["steps"][3]["start"] = fecha_inicio jsonEval["steps"][3]["end"] = fecha_fin return jsonEval
def cerrar_sesion_claro_drive(webdriver_test_ux: webdriver, jsonEval): tiempo_step_inicio = Temporizador.obtener_tiempo_timer() fecha_inicio = Temporizador.obtener_fecha_tiempo_actual() try: webdriver_test_ux.refresh() boton_ajustes = WebDriverWait(webdriver_test_ux, 10).until(EC.element_to_be_clickable((By.ID, 'expand'))) boton_ajustes.click() boton_cerrar_sesion = WebDriverWait(webdriver_test_ux, 20).until( EC.element_to_be_clickable((By.XPATH, '//li[@data-id="logout"]/a'))) boton_cerrar_sesion.click() WebDriverWait(webdriver_test_ux, 10).until(EC.presence_of_element_located((By.ID, 'login'))) jsonEval["steps"][4]["output"][0]["status"] = jsonConst.SUCCESS jsonEval["steps"][4]["status"] = jsonConst.SUCCESS jsonEval["steps"][4]["output"][0]["output"] = 'Se cierra sesion correctamente' except NoSuchElementException as e: jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][4]["status"] = jsonConst.FAILED jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg) except ElementClickInterceptedException as e: jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][4]["status"] = jsonConst.FAILED jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg) except TimeoutException as e: jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][4]["status"] = jsonConst.FAILED jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg) except ElementNotInteractableException as e: jsonEval["steps"][4]["output"][0]["status"] = jsonConst.FAILED jsonEval["steps"][4]["status"] = jsonConst.FAILED jsonEval["steps"][4]["output"][0]["output"] = 'No fue posible realizar el cierre de sesion: {}'.format(e.msg) tiempo_step_final = Temporizador.obtener_tiempo_timer() - tiempo_step_inicio fecha_fin = Temporizador.obtener_fecha_tiempo_actual() jsonEval["steps"][4]["time"] = FormatUtils.truncar_float_cadena(tiempo_step_final) jsonEval["steps"][4]["start"] = fecha_inicio jsonEval["steps"][4]["end"] = fecha_fin return jsonEval
def __book_slots(driver: webdriver, slots: list) -> bool: for slot in slots: # Attempt to book slot logger.info(f"Attempting to reserve time slot...") driver.execute_script(slot) time.sleep(1) success_alert = driver.find_element_by_id("alertBookingSuccess") if success_alert.get_attribute("style") == "display: block;": # Booking successful logger.info(f"Booked time slot!") return True else: logger.info(f"Failed to book time slot...") driver.refresh() logger.info(f"None of the preferred time slots were available") return False
def solicitar_cita(driver: webdriver, context: CustomerProfile): driver.execute_script("enviar('solicitud');") for i in range(REFRESH_PAGE_CYCLES): resp_text = body_text(driver) if "Por favor, valide el Captcha para poder continuar" in resp_text: success = process_captcha(driver, context, partially=True) if not success: return None return solicitar_cita(driver, context) elif "Seleccione la oficina donde solicitar la cita" in resp_text: logging.info("Towns hit! :)") # Office selection: time.sleep(0.3) try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.ID, "btnSiguiente")) ) except TimeoutException: logging.error("Timed out waiting for offices to load") return None res = select_office(driver, context) if res is None: time.sleep(5) driver.refresh() continue btn = driver.find_element_by_id("btnSiguiente") btn.send_keys(Keys.ENTER) return True elif "En este momento no hay citas disponibles" in resp_text: time.sleep(5) driver.refresh() continue else: logging.info("No towns") return None
def auto_alert_accept(driver: webdriver): """Checking js-alert and accpet This method wrapped by try-catch block. Except: NoAlertPresentException return values: No exception = True, Exception occured = False """ try: result = driver.switch_to_alert() result.accept() return True except NoAlertPresentException as e: """ There was no js-alert""" print("There is no js-alert") print(e.__class__.__name__) print(e) driver.refresh() return False except Exception as e: """ Unexpected exception""" print("Unexpected except") assert e.__class__.__name__ == 'NameError' return False
def get_car_data(self, driver: webdriver, i: int) -> None: c = inspect.currentframe() try: WebDriverWait(driver, 5).until( EC.element_to_be_clickable( (By.XPATH, '/html/body/div[1]/div[4]/div[2]/div[5]/table/tbody/tr[' + str(i) + ']/td[5]/p/a[@class="a_list"]'))).click() except Exception as e: print('Can\'t click on car %d. Reason %s' % (i, e)) driver.refresh() if self.alert_present(driver): return False tabs = driver.window_handles try: driver.switch_to.window(tabs[1]) except Exception as e: print('Can\'t switch to new window. Reason %s' % e) self.download_images(self.get_img(driver)) car: Dict[str] = {} car['category'] = self.category car['category_url'] = self.category_url try: car['title'] = self.ko_translate( driver.find_element_by_xpath( '/html/body/div[1]/div[1]/h2').text, "en") car['title-seo'] = self.ko_translate( driver.find_element_by_xpath( '/html/body/div[1]/div[1]/h2').text, "en") except: car['title'] = '' driver.refresh() try: car['price'] = int( driver.find_element_by_xpath( '/html/body/div[1]/div[2]/div[2]/p/strong/em').text. replace(',', '')) * 9.10 except: car['price'] = '' driver.refresh() try: car['description'] = '<div class="paper-tit"><font style="vertical-align: inherit;"><font style="vertical-align: inherit;">Протокол осмотра / осмотра автомобилей, выставленных на аукцион</font></font></div>' el = WebDriverWait(driver, 5).until( EC.element_to_be_clickable(( By.CSS_SELECTOR, 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table' ))) if (len(el.get_attribute("outerHTML")) > 5000): i = 0 while (i <= int(len(el.get_attribute("outerHTML")) / 4999)): if (i == int(len(el.get_attribute("outerHTML")) / 4999)): car['description'] += self.rm_new_line( self.ko_translate( el.get_attribute("outerHTML") [i * 4999:len(el.get_attribute("outerHTML"))], "ru")) else: car['description'] += self.rm_new_line( self.ko_translate( el.get_attribute("outerHTML")[i * 4999:i + 1 * 4999], "ru")) i += 1 else: car['description'] += self.rm_new_line( self.ko_translate(el.get_attribute("outerHTML"), "ru")) except Exception as e: car['description'] += '' print(c.f_lineno) print('Can\'t get protocol view. Reason %s' % e) driver.refresh() try: car['description'] += '<h2 class="page-subtit mt60"><font style="vertical-align: inherit"><font style="vertical-align: inherit">Детали автомобиля</font></font></h2>' el = WebDriverWait(driver, 5).until( EC.element_to_be_clickable(( By.CSS_SELECTOR, 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div.vehicle-detail > div.vehicle-detail_bar > table.tbl-v02' ))) car['description'] += self.rm_new_line( self.ko_translate(el.get_attribute("outerHTML"), "ru")) except Exception as e: car['description'] += '' print(c.f_lineno) print('Can\'t get car details. Reason %s' % e) driver.refresh() try: car['description'] += '<h2 class="page-subtit mt60" id="view-status"><font style="vertical-align: inherit"><font style="vertical-align: inherit">Состояние кузова автомобиля</font></font></h2>' el = WebDriverWait(driver, 5).until( EC.element_to_be_clickable(( By.CSS_SELECTOR, 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div.tab-status' ))) car['description'] += self.rm_new_line( self.ko_translate(el.get_attribute("outerHTML"), "ru")) except Exception as e: car['description'] += '' print(c.f_lineno) print('Can\'t get car condition. Reason %s' % e) driver.refresh() try: car['description'] += '<h2 class="page-subtit mt60"><font style="vertical-align: inherit;"><font style="vertical-align: inherit;">Видео автомобиля</font></font></h2>' el = WebDriverWait(driver, 5).until( EC.element_to_be_clickable(( By.CSS_SELECTOR, 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div.vehicle-detail-view > div:nth-child(5)' ))) car['description'] += self.rm_new_line( self.ko_translate(el.get_attribute("outerHTML"), "ru")) except Exception as e: car['description'] += '' print(c.f_lineno) print('Can\'t get car control list. Reason %s' % e) driver.refresh() try: year = int( driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table > tbody > tr:nth-child(5) > td:nth-child(4)' ).text) except Exception as e: print('Can\'t get car year. Reason %s' % e) driver.refresh() try: mark = self.ko_translate( driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > div > div > table > tbody > tr:nth-child(4) > td:nth-child(5)' ).text, "en") except Exception as e: print('Can\'t get car mark. Reason %s' % e) driver.refresh() try: color = self.ko_translate( driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(3) > td:nth-child(4)' ).text, "ru") except Exception as e: print('Can\'t get car color. Reason %s' % e) driver.refresh() try: fulel_data = { "가솔린": "Бензин", "디젤": "Дизель", "LPG": "LPG", "LPI하이브리드": "LPG гибрид", "가솔린하이브리드": "Бензиновый гибрид", "디젤하이브리드": "Дизельный гибрид", "전기": "Электрокар", "가솔린/LPG": "Бензин/LPG" } fuel = fulel_data[driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(4) > td:nth-child(4)' ).text] except Exception as e: fuel = "Дизель" print('Can\'t get car fuel. Reason %s' % e) driver.refresh() try: res = re.findall( "\d+", driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(5) > td:nth-child(4)' ).text) displacement = int(''.join(res)) except Exception as e: print('Can\'t get car displacement. Reason %s' % e) driver.refresh() try: transmission = driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(2) > td:nth-child(4)' ).text == "자동" if "Автомат" else "Механика" except Exception as e: print('Can\'t get car transmission. Reason %s' % e) driver.refresh() try: car_type = self.ko_translate( driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(6) > td:nth-child(2)' ).text, "ru") except Exception as e: print('Can\'t get car type. Reason %s' % e) driver.refresh() try: lot_number = self.ko_translate( driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-tit > p > strong' ).text, "ru") except Exception as e: print('Can\'t get car type. Reason %s' % e) driver.refresh() try: r = re.findall( "\d+", driver.find_element_by_css_selector( 'body > div.page-popup.exhibited-vehicle > div.vehicle-detail > div > div.vehicle-detail > div > table > tbody > tr:nth-child(1) > td:nth-child(4)' ).text) distance_driven = int(''.join(r)) except Exception as e: print('Can\'t get car distance driven. Reason %s' % e) driver.refresh() car['images'] = self.get_img_str(driver) car['count'] = '0' car['activation'] = '1' car['currency'] = 'USD' car['recomended'] = '0' car['new'] = '0' car['weight'] = '0' car['article'] = lot_number car['properties'] = ( 'Цвет=[type=assortmentCheckBox value=%s product_margin=Желтый|Белый|Серебро|Красный|Фиолетовый|Оранжевый|Зеленый|Серый|Золото|Коричневый|Голубой|Черный|Бежевый]&Кузов=[type=assortmentCheckBox value=%s product_margin=Универсал|Фургон|Фура|Трактор|Седан|Родстер|Пикап|Мотоцикл|Минивен|Хэтчбек|Кроссовер|Купе|Кабриолет|Багги]&Пробег=%d&Двигатель=%d&Год=%d&Трансмиссия=[type=assortmentCheckBox value=%s product_margin=Механика|Автомат]&Топливо=[type=assortmentCheckBox value=%s product_margin=Дизель|Бензин|Газ]&Модель=%s&Марка=%s&Номер лота=%s&Аукцион=lotteautoauction' % (color, car_type, distance_driven, displacement, year, transmission, fuel, mark, self.category, lot_number)) try: driver.close() driver.switch_to.window(tabs[0]) except Exception as e: print('Can\'t switch to old window. Reason %s' % e) self.write_csv(car)
def cycle_cita(driver: webdriver, context: CustomerProfile): driver.delete_all_cookies() try: driver.execute_script("window.localStorage.clear();") driver.execute_script("window.sessionStorage.clear();") except Exception as e: logging.error(e) pass if context.fast_forward_url: while True: try: driver.set_page_load_timeout(300 if context.first_load else 50) driver.get(context.fast_forward_url) except TimeoutException: logging.error("Timed out loading initial page") continue break context.first_load = False session_id = driver.get_cookie("JSESSIONID").get("value") logging.info(session_id) else: driver.get( "https://sede.administracionespublicas.gob.es/icpplus/index.html") time.sleep(1) # Let the user actually see something! # Select "Barcelona" select = Select(driver.find_element_by_id("form")) select.select_by_visible_text(context.city) btn = driver.find_element_by_id("btnAceptar") btn.send_keys(Keys.ENTER) # 2. Tramite selection: try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.ID, "tramiteGrupo[1]"))) except TimeoutException: logging.error("Timed out waiting for tramite to load") return None select = Select(driver.find_element_by_id("tramiteGrupo[1]")) # Select "Huellos" select.select_by_value(context.operation_code.value) btn = driver.find_element_by_id("btnAceptar") btn.send_keys(Keys.ENTER) # 3. Instructions page: try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.ID, "btnEntrar"))) except TimeoutException: logging.error("Timed out waiting for Instructions page to load") return None btn = driver.find_element_by_id("btnEntrar") btn.send_keys(Keys.ENTER) # 4. Data form: success = False if context.operation_code == OperationType.TOMA_HUELLAS: success = toma_huellas_step2(driver, context) elif context.operation_code == OperationType.RECOGIDA_DE_TARJETA: success = recogida_de_tarjeta_step2(driver, context) elif context.operation_code == OperationType.SOLICITUD: success = solicitud_step2(driver, context) if not success: return None try: wait_exact_time(driver, context) except TimeoutException: logging.error("Timed out waiting for exact time") return None # 5. Solicitar cita: btn = driver.find_element_by_id("btnEnviar") btn.send_keys(Keys.ENTER) for i in range(REFRESH_PAGE_CYCLES): try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.TAG_NAME, "body"))) except TimeoutException: logging.error("Timed out waiting for body to load") return None resp_text = driver.find_element_by_tag_name("body").text if "Seleccione la oficina donde solicitar la cita" in resp_text: logging.info("Towns hit! :)") # 6. Office selection: time.sleep(0.3) try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.ID, "btnSiguiente"))) except TimeoutException: logging.error("Timed out waiting for offices to load") return None res = select_office(driver, context) if res is None: time.sleep(5) driver.refresh() continue btn = driver.find_element_by_id("btnSiguiente") btn.send_keys(Keys.ENTER) break elif "En este momento no hay citas disponibles" in resp_text: time.sleep(5) driver.refresh() continue else: logging.info("No towns") return None # 7. phone-mail: try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.ID, "emailDOS"))) logging.info("Email page hit") except TimeoutException: logging.error("Timed out waiting for phone/email to load") return None element = driver.find_element_by_id("txtTelefonoCitado") element.send_keys(context.phone) # phone num element = driver.find_element_by_id("emailUNO") element.send_keys(context.email) element = driver.find_element_by_id("emailDOS") element.send_keys(context.email) btn = driver.find_element_by_id("btnSiguiente") btn.send_keys(Keys.ENTER) return cita_selection(driver, context)