def fetch(self, request: ScraperRequest) -> Optional[str]:
gid = request.center_info.internal_id
platform = request.center_info.plateforme
center_id = gid.split(platform)[-1]
start_date = request.get_start_date()
self.lieu = Lieu(
plateforme=Plateforme[PLATFORM.upper()],
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=f"mesoigner{request.internal_id}",
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
)
centre_api_url = MESOIGNER_APIs.get("slots",
"").format(id=center_id,
start_date=start_date)
response = self._client.get(centre_api_url, headers=MESOIGNER_HEADERS)
request.increase_request_count("slots")
if response.status_code == 403:
request.increase_request_count("error")
raise Blocked403(PLATFORM, centre_api_url)
response.raise_for_status()
rdata = response.json()
first_availability = self.get_appointments(request, rdata)
if self.lieu and first_availability is None:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
return first_availability
def fetch_slots(request: ScraperRequest, creneau_q=DummyQueue()):
if "keldoc.com" in request.url:
logger.debug(f"Fixing wrong hostname in request: {request.url}")
request.url = request.url.replace("keldoc.com",
"vaccination-covid.keldoc.com")
if not PLATFORM_ENABLED:
return None
center = KeldocCenter(request, client=session, creneau_q=creneau_q)
center.vaccine_motives = filter_vaccine_motives(center.appointment_motives)
center.lieu = Lieu(
plateforme=Plateforme[PLATFORM.upper()],
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=f"keldoc{request.internal_id}",
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
atlas_gid=request.atlas_gid,
)
# Find the first availability
date, count = center.find_first_availability(request.get_start_date())
if not date and center.lieu:
if center.lieu:
center.found_creneau(
PasDeCreneau(lieu=center.lieu,
phone_only=request.appointment_by_phone_only))
request.update_appointment_count(0)
return None
request.update_appointment_count(count)
return date.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
def get_appointments(self, request: ScraperRequest, slots_api):
first_availability = None
if len(slots_api.get("slots", [])) == 0:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
return None
for creneau in slots_api.get("slots", []):
dose_ranks = get_possible_dose_numbers(creneau["vaccine_name"])
self.found_creneau(
Creneau(
horaire=dateutil.parser.parse(creneau["datetime"]),
reservation_url=request.url,
dose=dose_ranks,
type_vaccin=get_vaccine_name(creneau["vaccine_name"]),
lieu=self.lieu,
)
)
request.update_appointment_count(request.appointment_count + 1)
if first_availability is None or creneau["datetime"] < first_availability:
first_availability = creneau["datetime"]
request.add_vaccine_type(get_vaccine_name(creneau["vaccine_name"]))
return first_availability
def fetch(self, request: ScraperRequest) -> Optional[str]:
result = self._fetch(request)
if result is None and self.lieu:
self.found_creneau(
PasDeCreneau(lieu=self.lieu,
phone_only=request.appointment_by_phone_only))
return result
def fetch(self, request: ScraperRequest) -> Optional[str]:
gid = request.center_info.internal_id
platform = request.center_info.plateforme
center_id = gid.split(platform)[-1]
start_date = datetime.date.today()
end_date = start_date + datetime.timedelta(NUMBER_OF_SCRAPED_DAYS)
self.lieu = Lieu(
plateforme=Plateforme[PLATFORM.upper()],
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=request.internal_id,
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
)
centre_api_url = BIMEDOC_APIs.get("slots", "").format(
pharmacy_id=center_id, start_date=start_date, end_date=end_date
)
response = self._client.get(centre_api_url, headers=BIMEDOC_HEADERS)
request.increase_request_count("slots")
if response.status_code == 403:
request.increase_request_count("error")
self.found_creneau(PasDeCreneau(lieu=self.lieu))
raise Blocked403(PLATFORM, centre_api_url)
response.raise_for_status()
rdata = response.json()
if not rdata:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
first_availability = self.get_appointments(request, rdata)
if self.lieu and first_availability is None:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
return first_availability
def fetch(
self,
request: ScraperRequest,
):
first_availability = None
profile = self.get_profile(request=request)
if not profile:
return None
self.lieu = Lieu(
plateforme=Plateforme.ORDOCLIC,
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=f"ordoclic{request.internal_id}",
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
)
entityId = profile["entityId"]
attributes = profile.get("attributeValues")
for settings in attributes:
if settings["label"] == "booking_settings" and settings["value"].get("option", "any") == "any":
request.set_appointments_only_by_phone(True)
return None
for professional in profile["publicProfessionals"]:
medicalStaffId = professional["id"]
reasons = get_reasons(entityId, request=request)
for reason in reasons["reasons"]:
if not is_reason_valid(reason):
continue
vaccine = get_vaccine_name(reason.get("name", ""))
dose = get_dose_number(reason)
request.add_vaccine_type(vaccine)
reasonId = reason["id"]
date_obj = datetime.strptime(request.get_start_date(), "%Y-%m-%d")
end_date = (date_obj + timedelta(days=NUMBER_OF_SCRAPED_DAYS)).strftime("%Y-%m-%d")
slots = self.get_slots(entityId, medicalStaffId, reasonId, request.get_start_date(), end_date, request)
date = self.parse_ordoclic_slots(request, slots, vaccine, dose)
if date is None:
continue
if first_availability is None or date < first_availability:
first_availability = date
if first_availability is None:
if self.lieu:
self.found_creneau(PasDeCreneau(lieu=self.lieu, phone_only=request.appointment_by_phone_only))
return None
return first_availability.isoformat()
def fetch(self, request, client):
url = request.get_url()
slug = url.split("/")[-1]
organization = get_organization_slug(slug, client, request)
if organization is None:
return None
if "error" in organization:
logger.warning(organization["error"])
for speciality in organization["speciality"]:
request.update_practitioner_type(DRUG_STORE if speciality["id"] == 190 else GENERAL_PRACTITIONER)
organization_id = organization.get("id")
reasons = organization.get("consultationReasons")
if reasons is None:
logger.warning(f"unable to get reasons from organization {organization_id}")
return None
if not get_valid_reasons(reasons):
return None
first_availability = None
self.lieu = Lieu(
plateforme=Plateforme.AVECMONDOC,
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=request.internal_id,
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
)
for reason in get_valid_reasons(reasons):
start_date = isoparse(request.get_start_date())
end_date = start_date + timedelta(days=NUMBER_OF_SCRAPED_DAYS)
vaccine = get_vaccine_name(reason["reason"])
dose = get_vaccine_dose(reason["reason"])
request.add_vaccine_type(vaccine)
availabilities = get_availabilities(
reason["id"], reason["organizationId"], start_date, end_date, client, request
)
date, appointment_count = self.parse_availabilities(availabilities, request, vaccine, dose)
if date is None:
continue
request.appointment_count += appointment_count
if first_availability is None or first_availability > date:
first_availability = date
if first_availability is None:
if self.lieu:
self.found_creneau(PasDeCreneau(lieu=self.lieu, phone_only=request.appointment_by_phone_only))
return None
return first_availability.isoformat()
def fetch(self, request, client):
self.lieu = Lieu(
plateforme=Plateforme.MAPHARMA,
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=f"mapharma{request.internal_id}",
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
)
url = request.get_url()
# on récupère les paramètres c (id_campagne) & l (id_type)
params = dict(parse.parse_qsl(parse.urlsplit(url).query))
id_campagne = int(params.get("c"))
id_type = int(params.get("l"))
day_slots = {}
# certaines campagnes ont des dispos mais 0 doses
# si total_libres est à 0 c'est qu'il n'y a pas de vraies dispo
pharmacy, campagne = self.get_pharmacy_and_campagne(id_campagne, id_type)
if campagne is None or campagne["total_libres"] == 0:
return None
# l'api ne renvoie que 7 jours, on parse un peu plus loin dans le temps
start_date = date.fromisoformat(request.get_start_date())
for delta in range(0, NUMBER_OF_SCRAPED_DAYS, 6):
new_date = start_date + timedelta(days=delta)
slots = self.get_slots(id_campagne, id_type, new_date.isoformat(), client, request=request)
for day, day_slot in slots.items():
if day in day_slots:
continue
day_slots[day] = day_slot
if not day_slots:
return
day_slots.pop("first", None)
day_slots.pop("first_text", None)
first_availability, slot_count = self.parse_slots(
day_slots, start_date, request, get_vaccine_name(campagne["nom"])
)
request.update_appointment_count(slot_count)
request.update_practitioner_type(DRUG_STORE)
request.add_vaccine_type(get_vaccine_name(campagne["nom"]))
if first_availability is None:
if self.lieu:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
return None
return first_availability.isoformat()
def parse_ordoclic_slots(self, request: ScraperRequest, availability_data, vaccine, dose):
first_availability = None
if not availability_data:
return None
availabilities = availability_data.get("slots", None)
availability_count = 0
if type(availabilities) is list:
availability_count = len(availabilities)
request.update_appointment_count(request.appointment_count + availability_count)
if "nextAvailableSlotDate" in availability_data:
nextAvailableSlotDate = availability_data.get("nextAvailableSlotDate", None)
if nextAvailableSlotDate is not None:
first_availability = datetime.strptime(nextAvailableSlotDate, "%Y-%m-%dT%H:%M:%S%z")
first_availability += first_availability.replace(tzinfo=timezone("CET")).utcoffset()
return first_availability
if not dose:
dose = []
if availabilities is None:
return None
for slot in availabilities:
timeStart = slot.get("timeStart", None)
if not timeStart:
continue
date = datetime.strptime(timeStart, "%Y-%m-%dT%H:%M:%S%z")
if "timeStartUtcOffset" in slot:
timeStartUtcOffset = slot["timeStartUtcOffset"]
date += timedelta(minutes=timeStartUtcOffset)
self.found_creneau(
Creneau(
horaire=date,
reservation_url=request.url,
dose=[dose],
type_vaccin=[vaccine],
lieu=self.lieu,
)
)
if first_availability is None or date < first_availability:
first_availability = date
if self.lieu and first_availability is None:
self.found_creneau(PasDeCreneau(lieu=self.lieu))
return first_availability
def test_resource_par_departement__0_creneau():
# Given
departement = "07"
creneau = PasDeCreneau(lieu=centre_lamastre)
expected = {
"version":
1,
"last_updated":
expected_now.isoformat(),
"centres_indisponibles": [{
"departement": "07",
"nom": "CENTRE DE VACCINATION COVID - LAMASTRE",
"url":
"https://www.maiia.com/centre-de-vaccination/07270-lamastre/centre-de-vaccination-covid---lamastre?centerid=5fff1f61b1a1aa1cc204f203",
"location": {
"longitude": 4.5,
"latitude": 45.0,
"city": "Lamastre",
"cp": "07270"
},
"metadata": None,
"prochain_rdv": None,
"last_scan_with_availabilities": None,
"request_counts": None,
"plateforme": "Maiia",
"type": "vaccination-center",
"appointment_count": 0,
"internal_id": "maiia5fff1f61b1a1aa1cc204f203",
"vaccine_type": [],
"appointment_by_phone_only": False,
"erreur": None,
"atlas_gid": None,
}],
"centres_disponibles": [],
}
# When
actual = next(
ResourceParDepartement.from_creneaux([creneau],
departement=departement,
now=now))
# Then
assert actual.asdict(
)["centres_indisponibles"] == expected["centres_indisponibles"]
assert actual.asdict(
)["centres_disponibles"] == expected["centres_disponibles"]
def _fetch(
self, request: ScraperRequest,
creneau_q=DummyQueue()) -> Optional[str]:
if not PLATFORM_ENABLED:
return None
url = request.get_url()
start_date = request.get_start_date()
url_query = parse_qs(urlparse.urlparse(url).query)
if "centerid" not in url_query:
logger.warning(f"No centerId in fetch url: {url}")
return None
center_id = url_query["centerid"][0]
reasons = get_reasons(center_id, self._client, request=request)
if not reasons:
return None
self.lieu = Lieu(
plateforme=Plateforme[PLATFORM.upper()],
url=request.url,
location=request.center_info.location,
nom=request.center_info.nom,
internal_id=f"maiia{request.internal_id}",
departement=request.center_info.departement,
lieu_type=request.practitioner_type,
metadata=request.center_info.metadata,
atlas_gid=request.atlas_gid,
)
first_availability, slots_count = self.get_first_availability(
center_id,
start_date,
reasons,
client=self._client,
request=request)
if first_availability is None:
if self.lieu:
self.found_creneau(
PasDeCreneau(lieu=self.lieu,
phone_only=request.appointment_by_phone_only))
return None
for reason in reasons:
request.add_vaccine_type(get_vaccine_name(reason["name"]))
request.update_internal_id(f"maiia{center_id}")
request.update_appointment_count(slots_count)
return first_availability.isoformat()
def test_doctolib_sends_pas_de_creneau():
# Given
start_date = "2021-04-03"
base_url = "https://partners.doctolib.fr/centre-de-vaccinations-internationales/ville1/centre1?pid=practice-165752&enable_cookies_consent=1" # noqa
center_info = CenterInfo(departement="07",
nom="Mon Super Centre",
url=base_url)
scrap_request = ScraperRequest(base_url, start_date, center_info)
def app(request: httpx.Request) -> httpx.Response:
assert "User-Agent" in request.headers
if request.url.path == "/booking/centre1.json":
path = Path("tests", "fixtures", "doctolib", "basic-booking.json")
return httpx.Response(200,
json=json.loads(
path.read_text(encoding="utf-8")))
assert request.url.path == "/availabilities.json"
path = Path("tests", "fixtures", "doctolib",
"basic-availabilities.json")
no_availabilities = {"availabilities": [{"slots": []}]}
return httpx.Response(200, json=no_availabilities)
client = httpx.Client(transport=httpx.MockTransport(app))
q = SimpleQueue()
slots = DoctolibSlots(client=client, cooldown_interval=0, creneau_q=q)
# When
slots.fetch(scrap_request)
actual = []
while not q.empty():
actual.append(q.get())
# Then
assert len(actual) == 1
assert actual[0] == PasDeCreneau(lieu=Lieu(
departement="07",
plateforme=Plateforme.DOCTOLIB,
url=base_url,
nom="Mon Super Centre",
internal_id="doctolib123456789pid165752",
lieu_type="vaccination-center",
))
def test_resource_creneaux_quotidiens__2_creneau_with_custom_tags():
# Given
departement = "07"
tags = {
"all": [lambda c: True],
"arnm": [
lambda c: c.type_vaccin == Vaccine.MODERNA or c.type_vaccin ==
Vaccine.PFIZER
],
"adeno": [
lambda c: c.type_vaccin == Vaccine.ASTRAZENECA or c.type_vaccin ==
Vaccine.JANSSEN
],
}
creneaux = [
Creneau(
horaire=dateutil.parser.parse("2021-06-01T06:30:00.000Z"),
lieu=centre_lamastre,
reservation_url="https://some.url/reservation",
timezone=gettz("Europe/Paris"),
type_vaccin=Vaccine.ASTRAZENECA,
),
Creneau(
horaire=dateutil.parser.parse("2021-05-27T18:12:00.000Z"),
lieu=centre_saint_andeol,
reservation_url="https://some.url/reservation",
timezone=gettz("Europe/Paris"),
type_vaccin=Vaccine.MODERNA,
),
PasDeCreneau(lieu=centre_saint_andeol),
]
expected = {
"departement":
"07",
"creneaux_quotidiens": [
{
"date": "2021-05-26",
"total": 0,
"creneaux_par_lieu": []
},
{
"date":
"2021-05-27",
"total":
1,
"creneaux_par_lieu": [{
"lieu":
centre_saint_andeol.internal_id,
"creneaux_par_tag": [
{
"tag": "all",
"creneaux": 1
},
{
"tag": "arnm",
"creneaux": 1
},
{
"tag": "adeno",
"creneaux": 0
},
],
}],
},
{
"date": "2021-05-28",
"total": 0,
"creneaux_par_lieu": []
},
{
"date": "2021-05-29",
"total": 0,
"creneaux_par_lieu": []
},
{
"date": "2021-05-30",
"total": 0,
"creneaux_par_lieu": []
},
{
"date": "2021-05-31",
"total": 0,
"creneaux_par_lieu": []
},
{
"date":
"2021-06-01",
"total":
1,
"creneaux_par_lieu": [{
"lieu":
centre_lamastre.internal_id,
"creneaux_par_tag": [
{
"tag": "all",
"creneaux": 1
},
{
"tag": "arnm",
"creneaux": 0
},
{
"tag": "adeno",
"creneaux": 1
},
],
}],
},
{
"date": "2021-06-02",
"total": 0,
"creneaux_par_lieu": []
},
],
}
# When
actual = next(
ResourceCreneauxQuotidiens.from_creneaux(creneaux,
next_days=7,
departement=departement,
now=now,
tags=tags))
# Then
assert actual.asdict() == expected