def test_empty_db(self, database):
"""
an empty db should return false
"""
with connection_scope(database.engine) as con:
r = check_bra_record_exist(con, "CHABLAIS", datetime.now())
assert r is False
def test_multi_massif(self):
with connection_scope() as con:
r1 = persist_massif(
con,
"CHABLAIS",
{"name": "Haute-savoie", "number": "74"},
"Alpes du Nord",
)
r2 = persist_massif(
con,
"MONT-BLANC",
{"name": "Haute-savoie", "number": "74"},
"Alpes du Nord",
)
assert isinstance(r1, UUID)
assert isinstance(r2, UUID)
req = (
select([ZoneTable.c.z_id, DepartmentTable.c.d_id])
.select_from(ZoneTable.join(DepartmentTable).join(MassifTable))
.where(MassifTable.c.m_id == bindparam("massif"))
)
id1 = con.execute(req, massif=r1).first()
id2 = con.execute(req, massif=r2).first()
assert id1.z_id == id2.z_id
assert id1.d_id == id2.d_id
def import_flowcapt_station():
db = create_database_connections().engine
with connection_scope(db) as con:
with resource_stream("nivo_api", "cli/data/flowcapt.geojson") as fp:
gj = geojson.load(fp)
for station in gj.features:
persist_flowcapt_station(con, station)
def import_all_bra():
"""
Same as `import_bra` but we request from March 2016 to now.
"""
start_date = date(year=2016, month=3, day=10)
date_range = [
date.today() - timedelta(days=x)
for x in range(0, (date.today() - start_date).days + 1)
]
for d in date_range:
massif = ""
try:
bra_dates = get_bra_date(d)
with connection_scope() as con:
for massif, m_date in bra_dates.items():
if not check_bra_record_exist(con, massif, m_date):
xml = get_bra_xml(massif, m_date)
processed_bra = process_xml(con, xml)
persist_bra(con, processed_bra)
click.echo(f"Persist {massif.capitalize()}")
except Exception as e:
log.debug(e)
log.critical(
f"an error occured when processing massif {massif} for date {d}"
)
def find_and_replace_foreign_key_value(self) -> List[Dict]:
def replace_num_sta_by_column_name(line: Dict,
con: Connection) -> Dict:
"""
You have to know that some station have no id (yes...)
"""
nivo_sensor = int(line["nr_nivo_sensor"])
s = select([
SensorStationTable.c.nss_id
]).where(SensorStationTable.c.nss_meteofrance_id == nivo_sensor)
res = con.execute(s).first()
if res is None:
logger.warning(
f"No station have been found for id {nivo_sensor} creating an empty one."
)
res = create_new_unknown_nivo_sensor_station(nivo_sensor, con)
line["nr_nivo_sensor"] = res.nss_id
return line
with connection_scope() as con:
replaced_csv = list()
for line in self.cleaned_csv:
replaced_csv.append(replace_num_sta_by_column_name(line, con))
self.cleaned_csv = replaced_csv
return self.cleaned_csv
def import_nivo_sensor_station():
# this need refactor
res = requests.get(f"{Config.METEO_FRANCE_NIVO_BASE_URL}/postesNivo.json")
res.raise_for_status()
with connection_scope() as con:
with con.begin():
for feature in res.json()["features"]:
pointz = feature["geometry"]
pointz["coordinates"].append(
int(feature["properties"]["Altitude"]))
mf_id = (feature["properties"]["ID"]
if feature["properties"]["ID"] != "" else None)
ins = (insert(SensorStationTable).values(
**{
"nss_name":
feature["properties"]["Nom"],
"nss_meteofrance_id":
mf_id,
"the_geom":
func.ST_SetSRID(
func.ST_GeomFromGeoJSON(json.dumps(pointz)), 4326),
}).on_conflict_do_nothing(index_elements=["nss_name"]))
con.execute(ins)
inserted = (con.execute(
select([func.count(SensorStationTable.c.nss_id).label("count")
])).first().count)
click.echo(f"{inserted} sensor station imported")
def _get_bra_by_uuid_or_404(self, bra_id: UUID) -> _Element:
with connection_scope() as con:
query = select([BraRecordTable.c.br_raw_xml
]).where(BraRecordTable.c.br_id == bra_id)
res = con.execute(query).first()
if not res:
bra_api.abort(HTTPStatus.NOT_FOUND)
return res.br_raw_xml
def test_multi_insert(self):
with connection_scope() as con:
uuid_list = list()
for _ in range(5):
uuid_list.append(persist_zone(con, "this_is_a_test"))
for x in uuid_list:
assert isinstance(x, UUID)
assert all(x == uuid_list[0] for x in uuid_list)
def test_download_fail(self, database):
url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/nivo.20190812.csv"
responses.add(responses.GET, url, status=503)
with connection_scope(database.engine) as con:
with pytest.raises(HTTPError):
download_nivo(
NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)),
con)
def import_nivo(csv_file: ANivoCsv) -> None:
csv_file.normalize()
csv_file.find_and_replace_foreign_key_value()
with connection_scope() as con:
with con.begin():
ins = insert(NivoRecordTable).values(
csv_file.cleaned_csv
) # .on_conflict_do_nothing(index_elements=['nss_name'])
con.execute(ins)
def test_massif(self):
with connection_scope() as con:
r = persist_massif(
con,
"CHABLAIS",
{"name": "Haute-savoie", "number": "74"},
"Alpes du Nord",
)
assert isinstance(r, UUID)
def test_import_nivo(self, database):
with open(os.path.join(CURRENT_DIR,
"test_data/nivo.20190812.csv")) as f:
with connection_scope(database.engine) as con:
nivo_csv = DictReader(f, delimiter=";")
n = NivoCsv(
NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)),
con)
n.nivo_csv = nivo_csv
def test_create_new_sensor_station_fail(self):
"""
It should fail when two unknown station have the same name. non-idempotency is assumed (tech debt FTW)
"""
with connection_scope() as con:
with pytest.raises(IntegrityError):
r = create_new_unknown_nivo_sensor_station(10, con)
assert isinstance(r.nss_id, UUID)
create_new_unknown_nivo_sensor_station(10, con)
def check_nivo_doesnt_exist(nivo_date: date) -> bool:
with connection_scope() as con:
s = exists([
NivoRecordTable.c.nr_date
]).where(cast(NivoRecordTable.c.nr_date, Date) == nivo_date)
s = select([s.label("exists")])
does_nivo_already_exist = con.execute(s).first().exists
logger.debug(
f"does nivo for date {nivo_date.strftime('%d-%m-%Y')} already exist : {does_nivo_already_exist}"
)
return not does_nivo_already_exist
def import_last_nivo_data():
# setup
# get http://donneespubliques.meteofrance.fr/donnees_libres/Txt/Nivo/lastNivo.js
# get last nivo data if needed
# process it
# import it.
db = create_database_connections().engine
with connection_scope(db) as con:
last_nivo = get_last_nivo_date()
if check_nivo_doesnt_exist(con, last_nivo.nivo_date):
downloaded_nivo = download_nivo(last_nivo, con)
import_nivo(con, downloaded_nivo)
def test_wrong_file_normalize(self, database):
file = resource_filename(
"test",
"test_cli/test_nivo_record_helper/test_data/nivo.wrong_data.csv")
with connection_scope(database.engine) as con:
date = NivoDate(False, datetime.now())
a = NivoCsv(date, con, "http://test")
a.nivo_csv = DictReader(file, delimiter=";")
with pytest.raises(KeyError) as e:
a.normalize()
assert str(e.value) == "'nr_numer_sta'"
def test_archive_download(self, database):
url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/Archive/nivo.201701.csv.gz"
with open(os.path.join(CURRENT_DIR, "test_data/nivo.201701.csv.gz"),
"rb") as f:
responses.add(responses.GET,
url,
body=f.read(),
content_type="application/x-gzip")
with connection_scope(database.engine) as con:
r = download_nivo(
NivoDate(is_archive=True, nivo_date=date(2017, 1, 1)), con)
assert isinstance(r, ArchiveNivoCss)
assert r.nivo_date == date(2017, 1, 1)
def test_recent_nivo_download(self, database):
url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/nivo.20190812.csv"
with open(os.path.join(CURRENT_DIR,
"test_data/nivo.20190812.csv")) as f:
responses.add(responses.GET,
url,
body=f.read(),
content_type="text/plain")
with connection_scope(database.engine) as con:
r = download_nivo(
NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)), con)
assert isinstance(r, NivoCsv)
assert r.nivo_date == date(2019, 8, 12)
def test_fetch_wrong_file(self, database):
responses.add(responses.GET,
"http://example.com",
content_type="text/html",
status=302)
date = NivoDate(False, datetime.now())
with connection_scope(database.engine) as con:
a = NivoCsv(date, con, "http://example.com")
assert a.download_url == "http://example.com"
assert a.nivo_date == date.nivo_date
with pytest.raises(HTTPError) as ex:
a.fetch_and_parse()
assert str(ex.value) == "Cannot found Nivo record"
def test_fetch(self, database):
with resource_stream(
"test",
"test_cli/test_nivo_record_helper/test_data/nivo.wrong_data.csv"
) as f:
responses.add(responses.GET,
"http://test",
body=f.read(),
content_type="text/plain")
with connection_scope(database.engine) as con:
date = NivoDate(False, datetime.now())
a = NivoCsv(date, con, "http://test")
assert a.download_url == "http://test"
assert a.nivo_date == date.nivo_date
def test_expected_json(database):
test_f = Feature(
properties={
"id": "FBER1",
"site": "La Berarde",
"altitude": 2390,
"country": "France",
},
geometry={"type": "Point", "coordinates": [6.237082, 44.949944]},
)
with connection_scope(database.engine) as con:
persist_flowcapt_station(con, test_f)
res = con.execute(select([FlowCaptStationTable])).fetchall()
assert len(res) == 1
assert res[0].fcs_id == "FBER1"
def import_last_bra():
# Open data from meteo france via it's obscure rest api because they are not providing opendata for the current day
# They start providing the BRA one day after it's been released, which makes the app useless.
with connection_scope() as con:
for dept in DepartmentTable.get(con):
try:
dept = get_bra_by_dept_from_mf_rpc_api(dept.d_number)
for massif in dept:
xml = format_xml_from_mf_rpc(massif["corpsBulletin"])
processed_bra = process_xml(con, xml)
persist_bra(con, processed_bra)
except HTTPError as e:
log.critical(f"dept {dept.d_name} cannot be fetch no BRA")
continue
except Exception as e:
log.critical(
f"an error occured when processing dept {dept.d_name} for today"
)
def _inject_test_data(self):
with connection_scope() as con:
nss_id = uuid4()
con.execute(SensorStationTable.insert().values({
"nss_id":
nss_id,
"nss_name":
"test",
"nss_meteofrance_id":
1,
"the_geom":
"SRID=4326;POINT(1 1 1)",
}))
con.execute(NivoRecordTable.insert().values({
"nr_date":
date(2019, 1, 1),
"nr_nivo_sensor":
nss_id
}))
def import_massifs():
massif_json = requests.get(Config.BRA_BASE_URL + "/massifs.json").json()
with connection_scope() as con:
# the 4th element of the massif is useless, and there are no BRA for it.
for zone in massif_json[:4]:
for dept in zone["departements"]:
for massif in dept["massifs"]:
click.echo(f"Importing {massif}")
try:
persist_massif(
con,
massif,
{
"name": dept["nom_dep"],
"number": dept["num_dep"]
},
zone["zone"],
)
except ValueError:
log.warning(f"Do no import massif: {massif}")
def import_all_nivo_data():
# setup
# download from 2010 to now
# download all file (via http, better)
# process it
# import it
all_nivo_date = get_all_nivo_date()
log.info(f"Need to process {len(all_nivo_date)}")
db = create_database_connections().engine
with connection_scope(db) as con:
for nivo_date in all_nivo_date:
if check_nivo_doesnt_exist(con, nivo_date.nivo_date):
try:
log.info(
f"Processing for {nivo_date.nivo_date.strftime('%d-%m-%Y')}"
)
downloaded_nivo = download_nivo(nivo_date, con)
import_nivo(con, downloaded_nivo)
except Exception as e:
click.echo("Something bad append")
log.debug(e)
def import_last_bra():
click.echo("This function doesn't work anymore sorry")
sys.exit()
# This is obsolete.
db = create_database_connections().engine
with connection_scope(db) as con:
for dept in DepartmentTable.get(con):
try:
dept = get_bra_by_dept_from_mf_rpc_api(dept.d_number)
for massif in dept:
xml = format_xml_from_mf_rpc(massif["corpsBulletin"])
processed_bra = process_xml(con, xml)
persist_bra(con, processed_bra)
except HTTPError as e:
log.critical(f"dept {dept['d_name']} cannot be fetch no BRA")
log.debug(e)
continue
except Exception as e:
log.critical(
f"an error occured when processing dept {dept['d_name']} for today"
)
log.debug(e)
def import_bra(bra_date):
"""
* setup
* request https://donneespubliques.meteofrance.fr/donnees_libres/Pdf/BRA/bra.%Y%m%d.json with all the date from december 2016 to today
* if not 302 (302 means 404 at meteofrance ðŸ˜)
* for all the date in all the json, download the xml of bra
* process (download + post process)
* import
"""
bra_dates = get_bra_date(bra_date)
with connection_scope() as con:
for massif, m_date in bra_dates.items():
try:
if not check_bra_record_exist(con, massif, m_date):
xml = get_bra_xml(massif, m_date)
processed_bra = process_xml(con, xml)
persist_bra(con, processed_bra)
click.echo(f"Persist {massif.capitalize()}")
except Exception as e:
log.debug(e)
log.critical(
f"an error occured when processing massif {massif} for date {m_date}"
)
def test_insert_zone(self):
with connection_scope() as con:
r = persist_zone(con, "this_is_a_test")
assert isinstance(r, UUID)
def test_no_massif(self):
with connection_scope() as con:
with pytest.raises(ValueError) as e:
_get_massif_id("CHABLAIS", con)
assert e.value.args[0] == "Cannot found massif CHABLAIS in the db"
assert e.type is ValueError
def test_process_xml_work(self, bra_xml_parsed):
self._setup_test()
with connection_scope() as c:
p = process_xml(c, bra_xml_parsed)
assert isinstance(p, list)
