def _find_visit_motive_id(data: dict, visit_motive_category_id: list = None): """ Etant donnée une réponse à /booking/<centre>.json, renvoie le cas échéant l'ID du 1er motif de visite disponible correspondant à une 1ère dose pour la catégorie de motif attendue. """ relevant_motives = {} for visit_motive in data.get("data", {}).get("visit_motives", []): vaccine_name = repr(get_vaccine_name(visit_motive["name"])) # On ne gère que les 1ère doses (le RDV pour la 2e dose est en général donné # après la 1ère dose, donc les gens n'ont pas besoin d'aide pour l'obtenir). if not is_appointment_relevant(visit_motive["name"]): continue vaccine_name = get_vaccine_name(visit_motive["name"]) # If this motive isn't related to vaccination if not visit_motive.get("first_shot_motive") and vaccine_name != Vaccine.JANSSEN: continue # If it's not a first shot motive # TODO: filter system if not visit_motive.get("first_shot_motive") and vaccine_name != Vaccine.JANSSEN: continue # Si le lieu de vaccination n'accepte pas les nouveaux patients # on ne considère pas comme valable. if "allow_new_patients" in visit_motive and not visit_motive["allow_new_patients"]: continue # NOTE: 'visit_motive_category_id' agit comme un filtre. Il y a 2 cas : # * visit_motive_category_id=None : pas de filtre, et on veut les motifs qui ne # sont pas non plus rattachés à une catégorie # * visit_motive_category_id=<id> : filtre => on veut les motifs qui # correspondent à la catégorie en question. if visit_motive_category_id is None or visit_motive.get("visit_motive_category_id") in visit_motive_category_id: relevant_motives[visit_motive["id"]] = vaccine_name return relevant_motives
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT) -> Optional[str]: url = request.get_url() start_date = request.get_start_date() url_query = parse_qs(urlparse.urlparse(url).query) if "centerid" not in url_query: logger.warning(f"No centerId in fetch url: {url}") return None center_id = url_query["centerid"][0] reasons = get_reasons(center_id, client=client) if not reasons: return None first_availability, slots_count, appointment_schedules = get_first_availability( center_id, start_date, reasons, client=client ) if first_availability == None: return None for reason in reasons: request.add_vaccine_type(get_vaccine_name(reason["name"])) request.update_internal_id(f"maiia{center_id}") request.update_appointment_count(slots_count) request.update_appointment_schedules(appointment_schedules) return first_availability.isoformat()
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT) -> str: url = request.get_url() # on récupère les paramètres c (id_campagne) & l (id_type) params = dict(parse.parse_qsl(parse.urlsplit(url).query)) id_campagne = int(params.get('c')) id_type = int(params.get('l')) day_slots = {} # l'api ne renvoie que 7 jours, on parse un peu plus loin dans le temps start_date = date.fromisoformat(request.get_start_date()) for delta in range(0, 30, 6): new_date = start_date + timedelta(days=delta) slots = get_slots(id_campagne, id_type, new_date.isoformat(), client) for day, day_slot in slots.items(): if day in day_slots: continue day_slots[day] = day_slot if not day_slots: return day_slots.pop('first', None) day_slots.pop('first_text', None) first_availability, slot_count = parse_slots(day_slots) request.update_appointment_count(slot_count) request.update_practitioner_type(DRUG_STORE) request.update_internal_id(url.encode('utf8').hex()[40:][:8]) pharmacy, campagne = get_pharmacy_and_campagne(id_campagne, id_type) request.add_vaccine_type(get_vaccine_name(campagne['nom'])) if first_availability is None: return None return first_availability.isoformat()
def filter_vaccine_motives(session, selected_cabinet, id, vaccine_specialties, vaccine_cabinets, request: ScraperRequest = None): if not id or not vaccine_specialties or not vaccine_cabinets: return None motive_categories = [] vaccine_motives = [] for specialty in vaccine_specialties: for cabinet in vaccine_cabinets: if selected_cabinet is not None and cabinet != selected_cabinet: continue if request: request.increase_request_count("motives") try: motive_req = session.get( API_KELDOC_MOTIVES.format(id, specialty, cabinet)) except TimeoutException: continue motive_req.raise_for_status() motive_data = motive_req.json() motive_categories.extend(motive_data) for motive_cat in motive_categories: motives = motive_cat.get("motives", {}) for motive in motives: motive_name = motive.get("name", None) if not motive_name or not is_appointment_relevant(motive_name): continue motive_agendas = [ motive_agenda.get("id", None) for motive_agenda in motive.get("agendas", {}) ] vaccine_type = get_vaccine_name(motive_name) if vaccine_type is None: vaccine_type = get_vaccine_name(motive_cat.get("name")) vaccine_motives.append({ "id": motive.get("id", None), "vaccine_type": vaccine_type, "agendas": motive_agendas }) return vaccine_motives
def get_first_availability( center_id: str, request_date: str, reasons: [dict], client: httpx.Client = DEFAULT_CLIENT ) -> [Optional[datetime], int, dict]: date = isoparse(request_date).replace(tzinfo=None) start_date = date.isoformat() end_date = (date + timedelta(days=MAIIA_DAY_LIMIT)).isoformat() first_availability = None slots_count = 0 appointment_schedules = [] counts = {} counts["chronodose"] = 0 for n in INTERVAL_SPLIT_DAYS: counts[f"{n}_days"] = 0 for consultation_reason in reasons: consultation_reason_name_quote = quote(consultation_reason.get("name"), "") if "injectionType" in consultation_reason and consultation_reason["injectionType"] in ["FIRST"]: slots = get_slots(center_id, consultation_reason_name_quote, start_date, end_date, client=client) slot_availability = parse_slots(slots) if slot_availability == None: continue for n in ( INTERVAL_SPLIT_DAY for INTERVAL_SPLIT_DAY in INTERVAL_SPLIT_DAYS if INTERVAL_SPLIT_DAY <= MAIIA_DAY_LIMIT ): n_date = (isoparse(start_date) + timedelta(days=n, seconds=-1)).isoformat() counts[f"{n}_days"] += count_slots(slots, start_date, n_date) slots_count += len(slots) if get_vaccine_name(consultation_reason["name"]) in CHRONODOSE_VACCINES: n_date = (isoparse(start_date) + timedelta(days=2, seconds=-1)).isoformat() counts["chronodose"] += count_slots(slots, start_date, n_date) if first_availability == None or slot_availability < first_availability: first_availability = slot_availability start_date = (paris_tz.localize(date)).isoformat() n_date = (paris_tz.localize(date + timedelta(days=2, seconds=-1))).isoformat() appointment_schedules.append({ "name": "chronodose", "from": start_date, "to": n_date, "total": counts["chronodose"] }) for n in INTERVAL_SPLIT_DAYS: n_date = (paris_tz.localize(date + timedelta(days=n, seconds=-1))).isoformat() appointment_schedules.append({ "name": f'{n}_days', "from": start_date, "to": n_date, "total": counts[f'{n}_days'] }) logger.debug(f"appointment_schedules: {appointment_schedules}") return first_availability, slots_count, appointment_schedules
def maiia_center_to_csv(center: dict, root_center: dict) -> dict: if 'url' not in center: logger.warning(f'url not found - {center}') csv = dict() csv['gid'] = center.get('id')[:8] csv['nom'] = center.get('name') csv['rdv_site_web'] = f'{MAIIA_URL}{center["url"]}?centerid={center["id"]}' if 'pharmacie' in center['url']: csv['type'] = DRUG_STORE else: csv['type'] = VACCINATION_CENTER csv['vaccine_type'] = [] for consultation_reason in root_center['consultationReasons']: vaccine_name = get_vaccine_name(consultation_reason.get('name')) if vaccine_name and vaccine_name not in csv['vaccine_type']: csv['vaccine_type'].append(vaccine_name) if 'publicInformation' not in center: return csv if 'address' in center['publicInformation']: csv['com_insee'] = center['publicInformation']['address'].get( 'inseeCode', '') if len(csv['com_insee']) < 5: zip = center['publicInformation']['address'].get('zipCode') csv['com_insee'] = departementUtils.cp_to_insee(zip) csv['address'] = center['publicInformation']['address'].get( 'fullAddress') if 'location' in center['publicInformation']['address']: csv['long_coor1'] = center['publicInformation']['address'][ 'location']['coordinates'][0] csv['lat_coor1'] = center['publicInformation']['address'][ 'location']['coordinates'][1] elif 'locality' in center['publicInformation']['address'] \ and 'location' in center['publicInformation']['address']['locality']: csv['long_coor1'] = center['publicInformation']['address'][ 'locality']['location']['x'] csv['lat_coor1'] = center['publicInformation']['address'][ 'locality']['location']['y'] if 'officeInformation' in center['publicInformation']: csv['phone_number'] = format_phone_number( center['publicInformation']['officeInformation'].get( 'phoneNumber', '')) if 'openingSchedules' in center['publicInformation'][ 'officeInformation']: csv['business_hours'] = maiia_schedule_to_business_hours( center['publicInformation']['officeInformation'] ['openingSchedules']) return csv
def test_get_vaccine_name(): assert get_vaccine_name("Vaccination Covid -55ans suite à une première injection d'AZ (ARNm)") == Vaccine.ARNM assert get_vaccine_name("Vaccination ARN suite à une 1ere injection Astra Zeneca") == Vaccine.ARNM assert ( get_vaccine_name("Vaccination Covid de moins de 55ans (vaccin ARNm) suite à une 1ère injection d'AZ") == Vaccine.ARNM ) assert get_vaccine_name("Vaccination Covid +55ans AZ") == Vaccine.ASTRAZENECA assert get_vaccine_name("Vaccination Covid Pfizer") == Vaccine.PFIZER assert get_vaccine_name("Vaccination Covid Moderna") == Vaccine.MODERNA
def maiia_center_to_csv(center: dict, root_center: dict) -> dict: if "url" not in center: logger.warning(f"url not found - {center}") csv = dict() csv["gid"] = center.get("id")[:8] csv["nom"] = center.get("name") csv["rdv_site_web"] = f'{MAIIA_URL}{center["url"]}?centerid={center["id"]}' if "pharmacie" in center["url"]: csv["type"] = DRUG_STORE else: csv["type"] = VACCINATION_CENTER csv["vaccine_type"] = [] for consultation_reason in root_center["consultationReasons"]: vaccine_name = get_vaccine_name(consultation_reason.get("name")) if vaccine_name and vaccine_name not in csv["vaccine_type"]: csv["vaccine_type"].append(vaccine_name) if "publicInformation" not in center: return csv if "address" in center["publicInformation"]: zip = center["publicInformation"]["address"].get("zipCode") csv["com_cp"] = zip csv["com_insee"] = center["publicInformation"]["address"].get("inseeCode", "") if len(csv["com_insee"]) < 5: csv["com_insee"] = departementUtils.cp_to_insee(zip) csv["address"] = center["publicInformation"]["address"].get("fullAddress") if "location" in center["publicInformation"]["address"]: csv["long_coor1"] = center["publicInformation"]["address"]["location"]["coordinates"][0] csv["lat_coor1"] = center["publicInformation"]["address"]["location"]["coordinates"][1] elif ( "locality" in center["publicInformation"]["address"] and "location" in center["publicInformation"]["address"]["locality"] ): csv["long_coor1"] = center["publicInformation"]["address"]["locality"]["location"]["x"] csv["lat_coor1"] = center["publicInformation"]["address"]["locality"]["location"]["y"] if "officeInformation" in center["publicInformation"]: csv["phone_number"] = format_phone_number( center["publicInformation"]["officeInformation"].get("phoneNumber", "") ) if "openingSchedules" in center["publicInformation"]["officeInformation"]: csv["business_hours"] = maiia_schedule_to_business_hours( center["publicInformation"]["officeInformation"]["openingSchedules"] ) return csv
def test_scraper_request(): request = ScraperRequest("https://doctolib.fr/center/center-test", "2021-04-14") request.update_internal_id("d739") request.update_practitioner_type(GENERAL_PRACTITIONER) request.update_appointment_count(42) request.add_vaccine_type(get_vaccine_name("Injection pfizer 1ère dose")) assert request is not None assert request.internal_id == "d739" assert request.appointment_count == 42 assert request.vaccine_type == [Vaccine.PFIZER] result = ScraperResult(request, "Doctolib", "2021-04-14T14:00:00.0000") assert result.default() == { "next_availability": "2021-04-14T14:00:00.0000", "platform": "Doctolib", "request": request, }
def test_scraper_request(): request = ScraperRequest("https://doctolib.fr/center/center-test", "2021-04-14") request.update_internal_id("d739") request.update_practitioner_type(GENERAL_PRACTITIONER) request.update_appointment_count(42) request.add_vaccine_type(get_vaccine_name("Injection pfizer 1ère dose")) assert request is not None assert request.internal_id == "d739" assert request.appointment_count == 42 assert request.vaccine_type == ['Pfizer-BioNTech'] result = ScraperResult(request, 'Doctolib', '2021-04-14T14:00:00.0000') assert result.default() == { 'next_availability': '2021-04-14T14:00:00.0000', 'platform': 'Doctolib', 'request': request }
def filter_vaccine_motives(session, selected_cabinet, id, vaccine_specialties, vaccine_cabinets): if not id or not vaccine_specialties or not vaccine_cabinets: return None motive_categories = [] vaccine_motives = [] for specialty in vaccine_specialties: for cabinet in vaccine_cabinets: if selected_cabinet is not None and cabinet != selected_cabinet: continue try: motive_req = session.get( API_KELDOC_MOTIVES.format(id, specialty, cabinet)) except TimeoutException: continue motive_req.raise_for_status() motive_data = motive_req.json() motive_categories.extend(motive_data) for motive_cat in motive_categories: motives = motive_cat.get('motives', {}) for motive in motives: motive_name = motive.get('name', None) if not motive_name or not is_appointment_relevant(motive_name): continue motive_agendas = [ motive_agenda.get('id', None) for motive_agenda in motive.get('agendas', {}) ] vaccine_motives.append({ 'id': motive.get('id', None), 'vaccine_type': get_vaccine_name(motive_name), 'agendas': motive_agendas }) return vaccine_motives
def test_vaccine_name(): name = get_vaccine_name("", Vaccine.PFIZER) assert name == "Pfizer-BioNTech"
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT, opendata_file: str = MAPHARMA_OPEN_DATA_FILE) -> str: url = request.get_url() # on récupère les paramètres c (id_campagne) & l (id_type) params = dict(parse.parse_qsl(parse.urlsplit(url).query)) id_campagne = int(params.get("c")) id_type = int(params.get("l")) day_slots = {} # l'api ne renvoie que 7 jours, on parse un peu plus loin dans le temps start_date = date.fromisoformat(request.get_start_date()) for delta in range(0, MAPHARMA_SLOT_LIMIT, 6): new_date = start_date + timedelta(days=delta) slots = get_slots(id_campagne, id_type, new_date.isoformat(), client) for day, day_slot in slots.items(): if day in day_slots: continue day_slots[day] = day_slot if not day_slots: return day_slots.pop("first", None) day_slots.pop("first_text", None) first_availability, slot_count = parse_slots(day_slots) request.update_appointment_count(slot_count) request.update_practitioner_type(DRUG_STORE) pharmacy, campagne = get_pharmacy_and_campagne(id_campagne, id_type, opendata_file) request.add_vaccine_type(get_vaccine_name(campagne["nom"])) appointment_schedules = [] s_date = paris_tz.localize( isoparse(request.get_start_date()) + timedelta(days=0)) n_date = s_date + timedelta(days=CHRONODOSES["Interval"], seconds=-1) chronodoses = 0 if get_vaccine_name(campagne["nom"]) in CHRONODOSES["Vaccine"]: chronodoses = count_appointements(day_slots, s_date, n_date) appointment_schedules.append({ "name": "chronodose", "from": s_date.isoformat(), "to": n_date.isoformat(), "total": chronodoses }) for n in INTERVAL_SPLIT_DAYS: n_date = s_date + timedelta(days=n, seconds=-1) appointment_schedules.append({ "name": f"{n}_days", "from": s_date.isoformat(), "to": n_date.isoformat(), "total": count_appointements(day_slots, s_date, n_date), }) logger.debug(f"appointment_schedules: {appointment_schedules}") request.update_appointment_schedules(appointment_schedules) if first_availability is None: return None return first_availability.isoformat()
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT): first_availability = None if not ORDOCLIC_ENABLED: return first_availability profile = get_profile(request, client) if not profile: return None slug = profile["profileSlug"] entityId = profile["entityId"] attributes = profile.get("attributeValues") for settings in attributes: if settings["label"] == "booking_settings" and settings["value"].get( "option", "any") == "any": request.set_appointments_only_by_phone(True) return None # create appointment_schedules array with names and dates appointment_schedules = [] start_date = paris_tz.localize( isoparse(request.get_start_date()) + timedelta(days=0)) end_date = start_date + timedelta(days=CHRONODOSES["Interval"], seconds=-1) appointment_schedules.append({ "name": "chronodose", "from": start_date.isoformat(), "to": end_date.isoformat(), "total": 0 }) for n in INTERVAL_SPLIT_DAYS: end_date = start_date + timedelta(days=n, seconds=-1) appointment_schedules.append({ "name": f"{n}_days", "from": start_date.isoformat(), "to": end_date.isoformat(), "total": 0 }) for professional in profile["publicProfessionals"]: medicalStaffId = professional["id"] name = professional["fullName"] zip = professional["zip"] reasons = get_reasons(entityId, request=request) for reason in reasons["reasons"]: if not is_reason_valid(reason): continue request.add_vaccine_type(get_vaccine_name(reason.get("name", ""))) reasonId = reason["id"] date_obj = datetime.strptime(request.get_start_date(), "%Y-%m-%d") end_date = (date_obj + timedelta(days=50)).strftime("%Y-%m-%d") slots = get_slots(entityId, medicalStaffId, reasonId, request.get_start_date(), end_date, client, request) date = parse_ordoclic_slots(request, slots) if date is None: continue # add counts to appointment_schedules availabilities = slots.get("slots", None) for i in range(0, len(appointment_schedules)): start_date = isoparse(appointment_schedules[i]["from"]) end_date = isoparse(appointment_schedules[i]["to"]) # do not count chronodose if wrong vaccine if (appointment_schedules[i]["name"] == "chronodose" and get_vaccine_name(reason.get( "name", "")) not in CHRONODOSES["Vaccine"]): continue appointment_schedules[i]["total"] += count_appointements( availabilities, start_date, end_date) request.update_appointment_schedules(appointment_schedules) logger.debug(f"appointment_schedules: {appointment_schedules}") if first_availability is None or date < first_availability: first_availability = date request.update_appointment_schedules(appointment_schedules) if first_availability is None: return None logger.debug(f"appointment_schedules: {request.appointment_schedules}") return first_availability.isoformat()