def get_agencies(cls, truncate=True):
"""
Get a list of agencies
"""
with Lock("agencies"):
request_params = {
'command': 'agencyList',
}
agencies_xml, api_call = cls.request(request_params, 'agency')
if not agencies_xml:
return []
db.session.begin()
if truncate:
db.session.query(Agency).delete()
agencies = []
for agency in agencies_xml:
region = Region.get_or_create(db.session,
title=agency.get('regionTitle'))
a = Agency.get_or_create(db.session,
tag = agency.get('tag'),
title = agency.get('title'),
short_title = agency.get('shortTitle'),
region = region,
api_call = api_call)
agencies.append(a)
db.session.commit()
return agencies
def get_agencies(cls, truncate=True):
"""
Get a list of agencies
"""
with Lock("agencies"):
request_params = {
'command': 'agencyList',
}
agencies_xml, api_call = cls.request(request_params, 'agency')
if not agencies_xml:
return []
db.session.begin()
if truncate:
db.session.query(Agency).delete()
agencies = []
for agency in agencies_xml:
region = Region.get_or_create(db.session,
title=agency.get('regionTitle'))
a = Agency.get_or_create(db.session,
tag=agency.get('tag'),
title=agency.get('title'),
short_title=agency.get('shortTitle'),
region=region,
api_call=api_call)
agencies.append(a)
db.session.commit()
return agencies
def main():
last_update = last_scrape()
print("Last update: {}".format(last_update))
print("Fetching page...")
data = scrape("https://www.indexsante.ca/urgences/", last_update)
cur_update = data.pop("last_update", last_update)
if last_update == cur_update:
print("Data already processed.")
return
all_records = []
scrape_id = Scrape.create(scrape_date=cur_update).id
defaults = {"scrape": scrape_id}
for (region_str, records) in data.items():
(region, _) = Region.get_or_create(nom=region_str, defaults=defaults)
for record in records:
hospital_name = record.pop("nom")
(hospital, _) = Hospital.get_or_create(nom=hospital_name,
region=region.id,
defaults=defaults)
record["hospital"] = hospital.id
record["scrape"] = scrape_id
all_records.append(record)
HospitalRecord.insert_many(all_records).execute()