def _find_visit_motive_id(data: dict, visit_motive_category_id: list = None):
"""
Etant donnée une réponse à /booking/<centre>.json, renvoie le cas échéant
l'ID du 1er motif de visite disponible correspondant à une 1ère dose pour
la catégorie de motif attendue.
"""
relevant_motives = {}
for visit_motive in data.get("data", {}).get("visit_motives", []):
vaccine_name = repr(get_vaccine_name(visit_motive["name"]))
# On ne gère que les 1ère doses (le RDV pour la 2e dose est en général donné
# après la 1ère dose, donc les gens n'ont pas besoin d'aide pour l'obtenir).
if not is_appointment_relevant(visit_motive["name"]):
continue
vaccine_name = get_vaccine_name(visit_motive["name"])
# If this motive isn't related to vaccination
if not visit_motive.get("first_shot_motive") and vaccine_name != Vaccine.JANSSEN:
continue
# If it's not a first shot motive
# TODO: filter system
if not visit_motive.get("first_shot_motive") and vaccine_name != Vaccine.JANSSEN:
continue
# Si le lieu de vaccination n'accepte pas les nouveaux patients
# on ne considère pas comme valable.
if "allow_new_patients" in visit_motive and not visit_motive["allow_new_patients"]:
continue
# NOTE: 'visit_motive_category_id' agit comme un filtre. Il y a 2 cas :
# * visit_motive_category_id=None : pas de filtre, et on veut les motifs qui ne
# sont pas non plus rattachés à une catégorie
# * visit_motive_category_id=<id> : filtre => on veut les motifs qui
# correspondent à la catégorie en question.
if visit_motive_category_id is None or visit_motive.get("visit_motive_category_id") in visit_motive_category_id:
relevant_motives[visit_motive["id"]] = vaccine_name
return relevant_motives
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT) -> Optional[str]:
url = request.get_url()
start_date = request.get_start_date()
url_query = parse_qs(urlparse.urlparse(url).query)
if "centerid" not in url_query:
logger.warning(f"No centerId in fetch url: {url}")
return None
center_id = url_query["centerid"][0]
reasons = get_reasons(center_id, client=client)
if not reasons:
return None
first_availability, slots_count, appointment_schedules = get_first_availability(
center_id, start_date, reasons, client=client
)
if first_availability == None:
return None
for reason in reasons:
request.add_vaccine_type(get_vaccine_name(reason["name"]))
request.update_internal_id(f"maiia{center_id}")
request.update_appointment_count(slots_count)
request.update_appointment_schedules(appointment_schedules)
return first_availability.isoformat()
def fetch_slots(request: ScraperRequest, client: httpx.Client = DEFAULT_CLIENT) -> str:
url = request.get_url()
# on récupère les paramètres c (id_campagne) & l (id_type)
params = dict(parse.parse_qsl(parse.urlsplit(url).query))
id_campagne = int(params.get('c'))
id_type = int(params.get('l'))
day_slots = {}
# l'api ne renvoie que 7 jours, on parse un peu plus loin dans le temps
start_date = date.fromisoformat(request.get_start_date())
for delta in range(0, 30, 6):
new_date = start_date + timedelta(days=delta)
slots = get_slots(id_campagne, id_type, new_date.isoformat(), client)
for day, day_slot in slots.items():
if day in day_slots:
continue
day_slots[day] = day_slot
if not day_slots:
return
day_slots.pop('first', None)
day_slots.pop('first_text', None)
first_availability, slot_count = parse_slots(day_slots)
request.update_appointment_count(slot_count)
request.update_practitioner_type(DRUG_STORE)
request.update_internal_id(url.encode('utf8').hex()[40:][:8])
pharmacy, campagne = get_pharmacy_and_campagne(id_campagne, id_type)
request.add_vaccine_type(get_vaccine_name(campagne['nom']))
if first_availability is None:
return None
return first_availability.isoformat()
def filter_vaccine_motives(session,
selected_cabinet,
id,
vaccine_specialties,
vaccine_cabinets,
request: ScraperRequest = None):
if not id or not vaccine_specialties or not vaccine_cabinets:
return None
motive_categories = []
vaccine_motives = []
for specialty in vaccine_specialties:
for cabinet in vaccine_cabinets:
if selected_cabinet is not None and cabinet != selected_cabinet:
continue
if request:
request.increase_request_count("motives")
try:
motive_req = session.get(
API_KELDOC_MOTIVES.format(id, specialty, cabinet))
except TimeoutException:
continue
motive_req.raise_for_status()
motive_data = motive_req.json()
motive_categories.extend(motive_data)
for motive_cat in motive_categories:
motives = motive_cat.get("motives", {})
for motive in motives:
motive_name = motive.get("name", None)
if not motive_name or not is_appointment_relevant(motive_name):
continue
motive_agendas = [
motive_agenda.get("id", None)
for motive_agenda in motive.get("agendas", {})
]
vaccine_type = get_vaccine_name(motive_name)
if vaccine_type is None:
vaccine_type = get_vaccine_name(motive_cat.get("name"))
vaccine_motives.append({
"id": motive.get("id", None),
"vaccine_type": vaccine_type,
"agendas": motive_agendas
})
return vaccine_motives
def get_first_availability(
center_id: str, request_date: str, reasons: [dict], client: httpx.Client = DEFAULT_CLIENT
) -> [Optional[datetime], int, dict]:
date = isoparse(request_date).replace(tzinfo=None)
start_date = date.isoformat()
end_date = (date + timedelta(days=MAIIA_DAY_LIMIT)).isoformat()
first_availability = None
slots_count = 0
appointment_schedules = []
counts = {}
counts["chronodose"] = 0
for n in INTERVAL_SPLIT_DAYS:
counts[f"{n}_days"] = 0
for consultation_reason in reasons:
consultation_reason_name_quote = quote(consultation_reason.get("name"), "")
if "injectionType" in consultation_reason and consultation_reason["injectionType"] in ["FIRST"]:
slots = get_slots(center_id, consultation_reason_name_quote, start_date, end_date, client=client)
slot_availability = parse_slots(slots)
if slot_availability == None:
continue
for n in (
INTERVAL_SPLIT_DAY
for INTERVAL_SPLIT_DAY in INTERVAL_SPLIT_DAYS
if INTERVAL_SPLIT_DAY <= MAIIA_DAY_LIMIT
):
n_date = (isoparse(start_date) + timedelta(days=n, seconds=-1)).isoformat()
counts[f"{n}_days"] += count_slots(slots, start_date, n_date)
slots_count += len(slots)
if get_vaccine_name(consultation_reason["name"]) in CHRONODOSE_VACCINES:
n_date = (isoparse(start_date) + timedelta(days=2, seconds=-1)).isoformat()
counts["chronodose"] += count_slots(slots, start_date, n_date)
if first_availability == None or slot_availability < first_availability:
first_availability = slot_availability
start_date = (paris_tz.localize(date)).isoformat()
n_date = (paris_tz.localize(date + timedelta(days=2, seconds=-1))).isoformat()
appointment_schedules.append({
"name": "chronodose",
"from": start_date,
"to": n_date,
"total": counts["chronodose"]
})
for n in INTERVAL_SPLIT_DAYS:
n_date = (paris_tz.localize(date + timedelta(days=n, seconds=-1))).isoformat()
appointment_schedules.append({
"name": f'{n}_days',
"from": start_date,
"to": n_date,
"total": counts[f'{n}_days']
})
logger.debug(f"appointment_schedules: {appointment_schedules}")
return first_availability, slots_count, appointment_schedules
def maiia_center_to_csv(center: dict, root_center: dict) -> dict:
if 'url' not in center:
logger.warning(f'url not found - {center}')
csv = dict()
csv['gid'] = center.get('id')[:8]
csv['nom'] = center.get('name')
csv['rdv_site_web'] = f'{MAIIA_URL}{center["url"]}?centerid={center["id"]}'
if 'pharmacie' in center['url']:
csv['type'] = DRUG_STORE
else:
csv['type'] = VACCINATION_CENTER
csv['vaccine_type'] = []
for consultation_reason in root_center['consultationReasons']:
vaccine_name = get_vaccine_name(consultation_reason.get('name'))
if vaccine_name and vaccine_name not in csv['vaccine_type']:
csv['vaccine_type'].append(vaccine_name)
if 'publicInformation' not in center:
return csv
if 'address' in center['publicInformation']:
csv['com_insee'] = center['publicInformation']['address'].get(
'inseeCode', '')
if len(csv['com_insee']) < 5:
zip = center['publicInformation']['address'].get('zipCode')
csv['com_insee'] = departementUtils.cp_to_insee(zip)
csv['address'] = center['publicInformation']['address'].get(
'fullAddress')
if 'location' in center['publicInformation']['address']:
csv['long_coor1'] = center['publicInformation']['address'][
'location']['coordinates'][0]
csv['lat_coor1'] = center['publicInformation']['address'][
'location']['coordinates'][1]
elif 'locality' in center['publicInformation']['address'] \
and 'location' in center['publicInformation']['address']['locality']:
csv['long_coor1'] = center['publicInformation']['address'][
'locality']['location']['x']
csv['lat_coor1'] = center['publicInformation']['address'][
'locality']['location']['y']
if 'officeInformation' in center['publicInformation']:
csv['phone_number'] = format_phone_number(
center['publicInformation']['officeInformation'].get(
'phoneNumber', ''))
if 'openingSchedules' in center['publicInformation'][
'officeInformation']:
csv['business_hours'] = maiia_schedule_to_business_hours(
center['publicInformation']['officeInformation']
['openingSchedules'])
return csv
def test_get_vaccine_name():
assert get_vaccine_name("Vaccination Covid -55ans suite à une première injection d'AZ (ARNm)") == Vaccine.ARNM
assert get_vaccine_name("Vaccination ARN suite à une 1ere injection Astra Zeneca") == Vaccine.ARNM
assert (
get_vaccine_name("Vaccination Covid de moins de 55ans (vaccin ARNm) suite à une 1ère injection d'AZ")
== Vaccine.ARNM
)
assert get_vaccine_name("Vaccination Covid +55ans AZ") == Vaccine.ASTRAZENECA
assert get_vaccine_name("Vaccination Covid Pfizer") == Vaccine.PFIZER
assert get_vaccine_name("Vaccination Covid Moderna") == Vaccine.MODERNA
def maiia_center_to_csv(center: dict, root_center: dict) -> dict:
if "url" not in center:
logger.warning(f"url not found - {center}")
csv = dict()
csv["gid"] = center.get("id")[:8]
csv["nom"] = center.get("name")
csv["rdv_site_web"] = f'{MAIIA_URL}{center["url"]}?centerid={center["id"]}'
if "pharmacie" in center["url"]:
csv["type"] = DRUG_STORE
else:
csv["type"] = VACCINATION_CENTER
csv["vaccine_type"] = []
for consultation_reason in root_center["consultationReasons"]:
vaccine_name = get_vaccine_name(consultation_reason.get("name"))
if vaccine_name and vaccine_name not in csv["vaccine_type"]:
csv["vaccine_type"].append(vaccine_name)
if "publicInformation" not in center:
return csv
if "address" in center["publicInformation"]:
zip = center["publicInformation"]["address"].get("zipCode")
csv["com_cp"] = zip
csv["com_insee"] = center["publicInformation"]["address"].get("inseeCode", "")
if len(csv["com_insee"]) < 5:
csv["com_insee"] = departementUtils.cp_to_insee(zip)
csv["address"] = center["publicInformation"]["address"].get("fullAddress")
if "location" in center["publicInformation"]["address"]:
csv["long_coor1"] = center["publicInformation"]["address"]["location"]["coordinates"][0]
csv["lat_coor1"] = center["publicInformation"]["address"]["location"]["coordinates"][1]
elif (
"locality" in center["publicInformation"]["address"]
and "location" in center["publicInformation"]["address"]["locality"]
):
csv["long_coor1"] = center["publicInformation"]["address"]["locality"]["location"]["x"]
csv["lat_coor1"] = center["publicInformation"]["address"]["locality"]["location"]["y"]
if "officeInformation" in center["publicInformation"]:
csv["phone_number"] = format_phone_number(
center["publicInformation"]["officeInformation"].get("phoneNumber", "")
)
if "openingSchedules" in center["publicInformation"]["officeInformation"]:
csv["business_hours"] = maiia_schedule_to_business_hours(
center["publicInformation"]["officeInformation"]["openingSchedules"]
)
return csv
def test_scraper_request():
request = ScraperRequest("https://doctolib.fr/center/center-test", "2021-04-14")
request.update_internal_id("d739")
request.update_practitioner_type(GENERAL_PRACTITIONER)
request.update_appointment_count(42)
request.add_vaccine_type(get_vaccine_name("Injection pfizer 1ère dose"))
assert request is not None
assert request.internal_id == "d739"
assert request.appointment_count == 42
assert request.vaccine_type == [Vaccine.PFIZER]
result = ScraperResult(request, "Doctolib", "2021-04-14T14:00:00.0000")
assert result.default() == {
"next_availability": "2021-04-14T14:00:00.0000",
"platform": "Doctolib",
"request": request,
}
def test_scraper_request():
request = ScraperRequest("https://doctolib.fr/center/center-test",
"2021-04-14")
request.update_internal_id("d739")
request.update_practitioner_type(GENERAL_PRACTITIONER)
request.update_appointment_count(42)
request.add_vaccine_type(get_vaccine_name("Injection pfizer 1ère dose"))
assert request is not None
assert request.internal_id == "d739"
assert request.appointment_count == 42
assert request.vaccine_type == ['Pfizer-BioNTech']
result = ScraperResult(request, 'Doctolib', '2021-04-14T14:00:00.0000')
assert result.default() == {
'next_availability': '2021-04-14T14:00:00.0000',
'platform': 'Doctolib',
'request': request
}
def filter_vaccine_motives(session, selected_cabinet, id, vaccine_specialties,
vaccine_cabinets):
if not id or not vaccine_specialties or not vaccine_cabinets:
return None
motive_categories = []
vaccine_motives = []
for specialty in vaccine_specialties:
for cabinet in vaccine_cabinets:
if selected_cabinet is not None and cabinet != selected_cabinet:
continue
try:
motive_req = session.get(
API_KELDOC_MOTIVES.format(id, specialty, cabinet))
except TimeoutException:
continue
motive_req.raise_for_status()
motive_data = motive_req.json()
motive_categories.extend(motive_data)
for motive_cat in motive_categories:
motives = motive_cat.get('motives', {})
for motive in motives:
motive_name = motive.get('name', None)
if not motive_name or not is_appointment_relevant(motive_name):
continue
motive_agendas = [
motive_agenda.get('id', None)
for motive_agenda in motive.get('agendas', {})
]
vaccine_motives.append({
'id':
motive.get('id', None),
'vaccine_type':
get_vaccine_name(motive_name),
'agendas':
motive_agendas
})
return vaccine_motives
def test_vaccine_name():
name = get_vaccine_name("", Vaccine.PFIZER)
assert name == "Pfizer-BioNTech"
def fetch_slots(request: ScraperRequest,
client: httpx.Client = DEFAULT_CLIENT,
opendata_file: str = MAPHARMA_OPEN_DATA_FILE) -> str:
url = request.get_url()
# on récupère les paramètres c (id_campagne) & l (id_type)
params = dict(parse.parse_qsl(parse.urlsplit(url).query))
id_campagne = int(params.get("c"))
id_type = int(params.get("l"))
day_slots = {}
# l'api ne renvoie que 7 jours, on parse un peu plus loin dans le temps
start_date = date.fromisoformat(request.get_start_date())
for delta in range(0, MAPHARMA_SLOT_LIMIT, 6):
new_date = start_date + timedelta(days=delta)
slots = get_slots(id_campagne, id_type, new_date.isoformat(), client)
for day, day_slot in slots.items():
if day in day_slots:
continue
day_slots[day] = day_slot
if not day_slots:
return
day_slots.pop("first", None)
day_slots.pop("first_text", None)
first_availability, slot_count = parse_slots(day_slots)
request.update_appointment_count(slot_count)
request.update_practitioner_type(DRUG_STORE)
pharmacy, campagne = get_pharmacy_and_campagne(id_campagne, id_type,
opendata_file)
request.add_vaccine_type(get_vaccine_name(campagne["nom"]))
appointment_schedules = []
s_date = paris_tz.localize(
isoparse(request.get_start_date()) + timedelta(days=0))
n_date = s_date + timedelta(days=CHRONODOSES["Interval"], seconds=-1)
chronodoses = 0
if get_vaccine_name(campagne["nom"]) in CHRONODOSES["Vaccine"]:
chronodoses = count_appointements(day_slots, s_date, n_date)
appointment_schedules.append({
"name": "chronodose",
"from": s_date.isoformat(),
"to": n_date.isoformat(),
"total": chronodoses
})
for n in INTERVAL_SPLIT_DAYS:
n_date = s_date + timedelta(days=n, seconds=-1)
appointment_schedules.append({
"name":
f"{n}_days",
"from":
s_date.isoformat(),
"to":
n_date.isoformat(),
"total":
count_appointements(day_slots, s_date, n_date),
})
logger.debug(f"appointment_schedules: {appointment_schedules}")
request.update_appointment_schedules(appointment_schedules)
if first_availability is None:
return None
return first_availability.isoformat()
def fetch_slots(request: ScraperRequest,
client: httpx.Client = DEFAULT_CLIENT):
first_availability = None
if not ORDOCLIC_ENABLED:
return first_availability
profile = get_profile(request, client)
if not profile:
return None
slug = profile["profileSlug"]
entityId = profile["entityId"]
attributes = profile.get("attributeValues")
for settings in attributes:
if settings["label"] == "booking_settings" and settings["value"].get(
"option", "any") == "any":
request.set_appointments_only_by_phone(True)
return None
# create appointment_schedules array with names and dates
appointment_schedules = []
start_date = paris_tz.localize(
isoparse(request.get_start_date()) + timedelta(days=0))
end_date = start_date + timedelta(days=CHRONODOSES["Interval"], seconds=-1)
appointment_schedules.append({
"name": "chronodose",
"from": start_date.isoformat(),
"to": end_date.isoformat(),
"total": 0
})
for n in INTERVAL_SPLIT_DAYS:
end_date = start_date + timedelta(days=n, seconds=-1)
appointment_schedules.append({
"name": f"{n}_days",
"from": start_date.isoformat(),
"to": end_date.isoformat(),
"total": 0
})
for professional in profile["publicProfessionals"]:
medicalStaffId = professional["id"]
name = professional["fullName"]
zip = professional["zip"]
reasons = get_reasons(entityId, request=request)
for reason in reasons["reasons"]:
if not is_reason_valid(reason):
continue
request.add_vaccine_type(get_vaccine_name(reason.get("name", "")))
reasonId = reason["id"]
date_obj = datetime.strptime(request.get_start_date(), "%Y-%m-%d")
end_date = (date_obj + timedelta(days=50)).strftime("%Y-%m-%d")
slots = get_slots(entityId, medicalStaffId, reasonId,
request.get_start_date(), end_date, client,
request)
date = parse_ordoclic_slots(request, slots)
if date is None:
continue
# add counts to appointment_schedules
availabilities = slots.get("slots", None)
for i in range(0, len(appointment_schedules)):
start_date = isoparse(appointment_schedules[i]["from"])
end_date = isoparse(appointment_schedules[i]["to"])
# do not count chronodose if wrong vaccine
if (appointment_schedules[i]["name"] == "chronodose"
and get_vaccine_name(reason.get(
"name", "")) not in CHRONODOSES["Vaccine"]):
continue
appointment_schedules[i]["total"] += count_appointements(
availabilities, start_date, end_date)
request.update_appointment_schedules(appointment_schedules)
logger.debug(f"appointment_schedules: {appointment_schedules}")
if first_availability is None or date < first_availability:
first_availability = date
request.update_appointment_schedules(appointment_schedules)
if first_availability is None:
return None
logger.debug(f"appointment_schedules: {request.appointment_schedules}")
return first_availability.isoformat()
