def normalize(site: dict, timestamp: str) -> dict:
links = [
schema.Link(authority="ct_gov", id=site["_id"]),
schema.Link(authority="ct_gov_network_id", id=site["networkId"]),
]
parent_organization = schema.Organization(name=site["networks"][0]["name"])
parsed_provider_link = provider_id_from_name(site["name"])
if parsed_provider_link is not None:
links.append(
schema.Link(authority=parsed_provider_link[0], id=parsed_provider_link[1])
)
parent_organization.id = parsed_provider_link[0]
return schema.NormalizedLocation(
id=f"ct_gov:{site['_id']}",
name=site["displayName"],
address=schema.Address(
street1=site["addressLine1"],
street2=site["addressLine2"],
city=site["city"],
state="CT",
zip=site["zip"],
),
location=_get_lat_lng(site),
contact=[
schema.Contact(
contact_type="booking", phone=site["phone"], website=site["link"]
),
],
languages=None,
opening_dates=None,
opening_hours=None,
availability=schema.Availability(
appointments=site["availability"],
),
inventory=[
schema.Vaccine(vaccine=vaccine["name"])
for vaccine in site["providerVaccines"]
],
access=schema.Access(
drive=site["isDriveThru"],
),
parent_organization=parent_organization,
links=links,
notes=None,
active=None,
source=schema.Source(
source="covidvaccinefinder_gov",
id=site["_id"],
fetched_from_uri="https://covidvaccinefinder.ct.gov/api/HttpTriggerGetProvider", # noqa: E501
fetched_at=timestamp,
published_at=site["lastModified"],
data=site,
),
).dict()
def normalize(site_blob: dict, timestamp: str) -> dict:
"""
sample entry:
{"providerId": 1013, "providerName": "\u1d42**York College - Health and Physical Education Complex - Queens", "vaccineBrand": "Pfizer", "address": "Jamaica, NY", "availableAppointments": "Y", "isShowable": true, "lastUpdated": "2021-04-23T20:04:24"} # noqa: E501
"""
name = NAME_CLEAN_RE.sub("", site_blob["providerName"]).strip()
city = CITY_RE.search(site_blob["address"]).group(1)
appts_available = True if site_blob[
"availableAppointments"] == "Y" else False
return schema.NormalizedLocation(
id=f"am_i_eligible_covid19vaccine_gov:{site_blob['providerId']}",
name=name,
address=schema.Address(
city=city,
state="NY",
),
availability=schema.Availability(appointments=appts_available),
inventory=_get_inventory(site_blob["vaccineBrand"]),
links=[
schema.Link(authority="am_i_eligible_covid19vaccine_gov",
id=site_blob["providerId"]),
],
source=_get_source(site_blob, timestamp),
).dict()
def normalize(site_blob: dict, timestamp: str) -> dict:
site = site_blob["properties"]
links = [schema.Link(authority="vaccinespotter_org", id=site["id"])]
parsed_provider_link = provider_id_from_name(
site["provider_brand_name"] # or use site["name"]?
)
if parsed_provider_link is not None:
links.append(
schema.Link(authority=parsed_provider_link[0],
id=parsed_provider_link[1]))
_strip_source_data(site_blob)
return schema.NormalizedLocation(
id=f"vaccinespotter_org:{site['id']}",
name=site["name"],
address=_get_address(site),
location=_get_lat_lng(site_blob["geometry"], site["id"]),
contact=[
schema.Contact(contact_type=None, phone=None, website=site["url"]),
],
languages=None,
opening_dates=None,
opening_hours=None,
availability=schema.Availability(
appointments=site["appointments_available"], drop_in=None),
inventory=None,
access=schema.Access(walk=None, drive=None, wheelchair=None),
parent_organization=None,
links=links,
notes=None,
active=None,
source=schema.Source(
source="vaccinespotter_org",
id=site["id"],
fetched_from_uri=
"https://www.vaccinespotter.org/api/v0/US.json", # noqa: E501
fetched_at=timestamp,
published_at=site["appointments_last_fetched"],
data=site_blob,
),
).dict()
def test_is_provider_tag_similar(full_location, minimal_location,
vial_location):
full_location.links.append(
location.Link(authority="_tag_provider", id="rite_aid"))
vial_location["properties"]["concordances"].append(
"_tag_provider:rite_aid")
assert match.is_provider_tag_similar(full_location, vial_location)
assert not match.is_provider_tag_similar(minimal_location, vial_location)
def _bulk_add_placekey_link(
locs: Collection[location.NormalizedLocation],
placekey_api: Optional[PlacekeyAPI],
) -> None:
"""Add placekey concordance to location if we can"""
if placekey_api is None:
return
def _placekey_record(loc: location.NormalizedLocation) -> Optional[dict]:
if not loc.location:
return None
if not _valid_address(loc):
return None
street_address = [loc.address.street1]
if loc.address.street2:
street_address.append(loc.address.street2)
return {
"latitude": loc.location.latitude,
"longitude": loc.location.longitude,
"location_name": loc.name,
"street_address": ", ".join(street_address),
"city": loc.address.city,
"region": loc.address.state,
"postal_code": loc.address.zip,
"iso_country_code": "US",
}
records = {}
for loc in locs:
record = _placekey_record(loc)
if record:
records[loc.id] = record
placekeys = placekey_api.lookup_placekeys(records)
if not placekeys:
return
for loc in locs:
placekey_id = placekeys.get(loc.id)
if not placekey_id:
continue
# Verify "what" part of placekey is specified or else placekey isn't specific enough
if "@" not in placekey_id or placekey_id.startswith("@"):
continue
loc.links = [
*(loc.links or []),
location.Link(authority="placekey", id=placekey_id),
]
def _get_links(site: dict) -> Optional[List[schema.Link]]:
links = None
parsed_provider_link = provider_id_from_name(site["title"])
if parsed_provider_link is not None:
provider_link = schema.Link(
authority=parsed_provider_link[0],
id=parsed_provider_link[1],
)
links = [provider_link]
return links
def _get_links(loc: GMVLocation) -> Optional[List[location.Link]]:
if not loc.external_ids:
return None
links = []
for provider_id, store_id in loc.external_ids:
if not store_id:
logger.info("Missing value for external_id %s", provider_id)
continue
authority = None
if provider_id == "vaccinespotter":
authority = "vaccinespotter_org"
elif provider_id == "vtrcks":
authority = location.LocationAuthority.VTRCKS
elif provider_id.startswith("univaf_"):
# Skip versioned univaf ids until we figure out what to do with them.
continue
elif provider_id in ("njiis_covid", "comassvax"):
# Skip because their ids are just addresses
continue
elif provider_id in ("centura_driveup_event", "denver_ball_arena"):
# Skip because I am not sure if the ids are of good quality or useful
continue
elif (provider_id.endswith("_powered_by_walgreens")
or provider_id.endswith("_a_walgreens_pharmacy")
or provider_id.endswith("_a_walgreens_rx")):
# These stores keep their custom names, but have walgreens store ids
authority = location.VaccineProvider.WALGREENS
else:
authority = PROVIDER_MAPPING.get(provider_id)
if not authority:
logger.info("Unrecognized provider for external_id %s:%s",
provider_id, store_id)
continue
links.append(location.Link(authority=authority, id=str(store_id)))
if not links:
return None
return links
def _add_source_link(loc: location.NormalizedLocation) -> None:
"""Add source link from source if missing"""
if not loc.source:
return
if not loc.source.source or not loc.source.id:
return
existing_links = _generate_link_map(loc)
if str(loc.source.source) in existing_links:
return
loc.links = [
*(loc.links or []),
location.Link(authority=loc.source.source, id=loc.source.id),
]
def _add_provider_tag(loc: location.NormalizedLocation) -> None:
"""Add provider tag to concordances to use for matching"""
if not loc.parent_organization:
return
if not loc.parent_organization.id:
return
existing_links = _generate_link_map(loc)
if PROVIDER_TAG in existing_links:
return
provider_id = str(loc.parent_organization.id)
loc.links = [
*(loc.links or []),
location.Link(authority=PROVIDER_TAG, id=provider_id),
]
def _add_provider_from_name(loc: location.NormalizedLocation) -> None:
"""Add provider link from name if missing"""
if not loc.name:
return
provider_tuple = normalize.provider_id_from_name(loc.name)
if not provider_tuple:
return
provider_authority, provider_id = provider_tuple
existing_links = _generate_link_map(loc)
if str(provider_authority) not in existing_links:
loc.links = [
*(loc.links or []),
location.Link(authority=provider_authority, id=provider_id),
]
if not loc.parent_organization:
loc.parent_organization = location.Organization(id=provider_authority)
def _get_links(site: dict) -> Optional[List[schema.Link]]:
maybe_provider = provider_id_from_name(site["name"])
if maybe_provider:
return [schema.Link(authority=maybe_provider[0], id=maybe_provider[1])]
return None
def test_invalid_authority(invalid_authority):
with pytest.raises(pydantic.error_wrappers.ValidationError):
location.Link(authority=invalid_authority, id="abc123")
def test_valid_authority(valid_authority):
assert location.Link(authority=valid_authority, id="abc123")
def full_location():
return location.NormalizedLocation(
id="source:dad13365-2202-4dea-9b37-535288b524fe",
name="Rite Aid Pharmacy 5952",
address=location.Address(
street1="1991 Mountain Boulevard",
city="Oakland",
state="CA",
zip="94611",
),
location=location.LatLng(
latitude=37.8273167,
longitude=-122.2105179,
),
contact=[
location.Contact(
contact_type=location.ContactType.BOOKING,
phone="(916) 445-2841",
),
location.Contact(
contact_type=location.ContactType.GENERAL,
phone="(510) 339-2215",
),
],
languages=["en"],
opening_dates=[
location.OpenDate(
opens="2021-04-01",
closes="2021-04-01",
),
],
opening_hours=[
location.OpenHour(
day=location.DayOfWeek.MONDAY,
opens="08:00",
closes="14:00",
),
],
availability=location.Availability(
drop_in=False,
appointments=True,
),
inventory=[
location.Vaccine(
vaccine=location.VaccineType.MODERNA,
supply_level=location.VaccineSupply.IN_STOCK,
),
],
access=location.Access(
walk=True,
drive=False,
wheelchair=location.WheelchairAccessLevel.PARTIAL,
),
parent_organization=location.Organization(
id=location.VaccineProvider.RITE_AID,
name="Rite Aid Pharmacy",
),
links=[
location.Link(
authority=location.LocationAuthority.GOOGLE_PLACES,
id="ChIJA0MOOYWHj4ARW8M-vrC9yGA",
),
],
notes=["long note goes here"],
active=True,
source=location.Source(
source="source",
id="dad13365-2202-4dea-9b37-535288b524fe",
fetched_from_uri="https://example.org",
fetched_at="2020-04-04T04:04:04.4444",
published_at="2020-04-04T04:04:04.4444",
data={"id": "dad13365-2202-4dea-9b37-535288b524fe"},
),
)
def normalize(site: dict, timestamp: str) -> dict:
address = site["location"]["address"]
address_parts = [p.strip() for p in address.split(",")]
# Remove city from end of address
address_parts.pop()
street1 = address_parts[0]
street2 = None
if len(address_parts) > 1:
street2 = ", ".join(address_parts[1:])
links = [schema.Link(authority="sf_gov", id=site["id"])]
parsed_provider_link = provider_id_from_name(site["name"])
if parsed_provider_link is not None:
links.append(
schema.Link(authority=parsed_provider_link[0],
id=parsed_provider_link[1]))
contacts = []
if site["booking"]["phone"] and site["booking"]["phone"].lower() != "none":
contacts.append(
schema.Contact(contact_type="booking",
phone=site["booking"]["phone"]))
if site["booking"]["url"] and site["booking"]["url"].lower() != "none":
contacts.append(
schema.Contact(contact_type="booking",
website=site["booking"]["url"]))
if site["booking"]["info"] and site["booking"]["info"].lower() != "none":
contacts.append(
schema.Contact(contact_type="booking",
other=site["booking"]["info"]))
return schema.NormalizedLocation(
id=f"sf_gov:{site['id']}",
name=site["name"],
address=schema.Address(
street1=street1,
street2=street2,
city=site["location"]["city"],
state="CA",
zip=(site["location"]["zip"] if site["location"]["zip"]
and site["location"]["zip"].lower() != "none" else None),
),
location=schema.LatLng(
latitude=site["location"]["lat"],
longitude=site["location"]["lng"],
),
contact=contacts,
languages=[k for k, v in site["access"]["languages"].items() if v],
opening_dates=None,
opening_hours=None,
availability=schema.Availability(
appointments=site["appointments"]["available"],
drop_in=site["booking"]["dropins"],
),
inventory=None,
access=schema.Access(
walk=site["access_mode"]["walk"],
drive=site["access_mode"]["drive"],
wheelchair="yes" if site["access"]["wheelchair"] else "no",
),
parent_organization=None,
links=links,
notes=None,
active=site["active"],
source=schema.Source(
source="sf_gov",
id=site["id"],
fetched_from_uri=
"https://vaccination-site-microservice.vercel.app/api/v1/appointments", # noqa: E501
fetched_at=timestamp,
published_at=site["appointments"]["last_updated"],
data=site,
),
).dict()
def test_valid_location():
# Minimal record
assert location.NormalizedLocation(
id="source:id",
source=location.Source(
source="source",
id="id",
data={"id": "id"},
),
)
# Full record with str enums
full_loc = location.NormalizedLocation(
id="source:id",
name="name",
address=location.Address(
street1="1991 Mountain Boulevard",
street2="#1",
city="Oakland",
state="CA",
zip="94611",
),
location=location.LatLng(
latitude=37.8273167,
longitude=-122.2105179,
),
contact=[
location.Contact(
contact_type="booking",
phone="(916) 445-2841",
)
],
languages=["en"],
opening_dates=[
location.OpenDate(
opens="2021-04-01",
closes="2021-04-01",
),
],
opening_hours=[
location.OpenHour(
day="monday",
opens="08:00",
closes="14:00",
),
],
availability=location.Availability(
drop_in=False,
appointments=True,
),
inventory=[
location.Vaccine(
vaccine="moderna",
supply_level="in_stock",
),
],
access=location.Access(
walk=True,
drive=False,
wheelchair="partial",
),
parent_organization=location.Organization(
id="rite_aid",
name="Rite Aid",
),
links=[
location.Link(
authority="google_places",
id="abc123",
),
],
notes=["note"],
active=True,
source=location.Source(
source="source",
id="id",
fetched_from_uri="https://example.org",
fetched_at="2020-04-04T04:04:04.4444",
published_at="2020-04-04T04:04:04.4444",
data={"id": "id"},
),
)
assert full_loc
# Verify dict serde
full_loc_dict = full_loc.dict()
assert full_loc_dict
parsed_full_loc = location.NormalizedLocation.parse_obj(full_loc_dict)
assert parsed_full_loc
assert parsed_full_loc == full_loc
# Verify json serde
full_loc_json = full_loc.json()
assert full_loc_json
parsed_full_loc = location.NormalizedLocation.parse_raw(full_loc_json)
assert parsed_full_loc
assert parsed_full_loc == full_loc
# Verify dict->json serde
full_loc_json_dumps = json.dumps(full_loc_dict)
assert full_loc_json_dumps
assert full_loc_json_dumps == full_loc_json
parsed_full_loc = location.NormalizedLocation.parse_raw(
full_loc_json_dumps)
assert parsed_full_loc
assert parsed_full_loc == full_loc
