def scrape_debug(urls): # pragma: no cover enable_logger_for_debug() start_date = get_start_date() for rdv_site_web in urls: rdv_site_web = fix_scrap_urls(rdv_site_web) logger.info("scraping URL %s", rdv_site_web) try: result = fetch_centre_slots(rdv_site_web, start_date) except Exception as e: logger.exception(f"erreur lors du traitement") logger.info( f'{result.platform!s:16} {result.next_availability or ""!s:32}')
def cherche_prochain_rdv_dans_centre( centre: dict) -> CenterInfo: # pragma: no cover center_data = CenterInfo.from_csv_data(centre) start_date = get_start_date() has_error = None result = None try: result = fetch_centre_slots(centre["rdv_site_web"], start_date, input_data=centre.get("booking")) center_data.fill_result(result) except ScrapeError as scrape_error: logger.error( f"erreur lors du traitement de la ligne avec le gid {centre['gid']} {str(scrape_error)}" ) has_error = scrape_error except CircuitBreakerOffException as error: logger.error( f"circuit '{error.name}' désactivé lors du traîtement de la ligne avec le gid {centre['gid']}: {str(error)}" ) has_error = error except Exception: logger.error( f"erreur lors du traitement de la ligne avec le gid {centre['gid']}" ) traceback.print_exc() if has_error is None: logger.info( f'{centre.get("gid", "")!s:>8} {center_data.plateforme!s:16} {center_data.prochain_rdv or ""!s:32} {center_data.departement!s:6}' ) else: logger.info( f'{centre.get("gid", "")!s:>8} {center_data.plateforme!s:16} {"Erreur" or ""!s:32} {center_data.departement!s:6}' ) if result is not None and result.request.url is not None: center_data.url = result.request.url.lower() if result.request.internal_id is None: center_data.internal_id = f'{result.platform.lower()}{centre.get("gid", "")}' if "type" in centre: center_data.type = centre["type"] if not center_data.type: center_data.type = VACCINATION_CENTER center_data.gid = centre.get("gid", "") logger.debug(center_data.default()) return center_data
def scrape_debug(urls): # pragma: no cover enable_logger_for_debug() start_date = get_start_date() for rdv_site_web in urls: rdv_site_web = fix_scrap_urls(rdv_site_web) logger.info("scraping URL %s", rdv_site_web) try: result = fetch_centre_slots(rdv_site_web, start_date) except Exception: logger.exception(f"erreur lors du traitement") logger.info( f'{result.platform!s:16} {result.next_availability or ""!s:32}') if result.request.appointment_count: logger.debug( f"appointments: {result.request.appointment_count}:\n{result.request.appointment_schedules}" ) log_requests(result.request)
def cherche_prochain_rdv_dans_centre( data: Tuple[dict, Queue]) -> CenterInfo: # pragma: no cover timestamp_before_request = datetime.datetime.now() centre, creneau_q = data center_data = CenterInfo.from_csv_data(centre) start_date = get_start_date() has_error = None result = None try: result = fetch_centre_slots( centre["rdv_site_web"], centre["platform_is"] if "platform_is" in centre.keys() else None, start_date, creneau_q=creneau_q, center_info=center_data, input_data=centre.get("booking"), atlas_gid=centre["atlas_gid"] if "atlas_gid" in centre.keys() else None, ) center_data.fill_result(result) except Blocked403 as blocked_doctolib__error: logger.error( f"erreur lors du traitement de la ligne avec le gid {centre['gid']} {str(blocked_doctolib__error)}" ) has_error = blocked_doctolib__error except DoublonDoctolib as doublon_doctolib: has_error = doublon_doctolib doublon = Color("{autored}DOUBLON{/autored}") gid = centre.get( "gid", "") if len(centre.get("gid", "")) < 50 else centre.get( "gid", "")[0:50] platform = "Doctolib" departement = center_data.departement if center_data.departement else "" table_data = [[ platform + str(" " * (15 - len(platform))), gid + str(" " * (50 - len(gid))), doublon + str(" " * (35 - len(doublon))), departement + str(" " * (3 - len(departement))), ]] table_instance = DoubleTable(table_data) table_instance.outer_border = False print(table_instance.table) except CircuitBreakerOffException as error: logger.error( f"circuit '{error.name}' désactivé lors du traîtement de la ligne avec le gid {centre['gid']}: {str(error)}" ) has_error = error except Exception: logger.error( f"erreur lors du traitement de la ligne avec le gid {centre['gid']}" ) traceback.print_exc() else: next_appointment = (Color("{autogreen}" + center_data.prochain_rdv + "{/autogreen}") if center_data.prochain_rdv else "") gid = centre.get( "gid", "") if len(centre.get("gid", "")) < 50 else centre.get( "gid", "")[0:50] platform = center_data.plateforme if center_data.plateforme else "Autre" departement = center_data.departement if center_data.departement else "" table_data = [[ platform + str(" " * (15 - len(platform))), gid + str(" " * (50 - len(gid))), next_appointment + str(" " * (35 - len(next_appointment))), departement + str(" " * (3 - len(departement))), ]] table_instance = DoubleTable(table_data) table_instance.outer_border = False print(table_instance.table) time_for_request = (datetime.datetime.now() - timestamp_before_request).total_seconds() if result is not None and result.request.url is not None: center_data.url = result.request.url.lower() if result.request.internal_id is None: center_data.internal_id = f'{result.platform.lower()}{centre.get("gid", "")}' if "type" in centre: center_data.type = centre["type"] if not center_data.type: center_data.type = VACCINATION_CENTER center_data.gid = centre.get("gid", "") center_data.time_for_request = time_for_request return center_data