def test_empty_db(self, database): """ an empty db should return false """ with connection_scope(database.engine) as con: r = check_bra_record_exist(con, "CHABLAIS", datetime.now()) assert r is False
def test_multi_massif(self): with connection_scope() as con: r1 = persist_massif( con, "CHABLAIS", {"name": "Haute-savoie", "number": "74"}, "Alpes du Nord", ) r2 = persist_massif( con, "MONT-BLANC", {"name": "Haute-savoie", "number": "74"}, "Alpes du Nord", ) assert isinstance(r1, UUID) assert isinstance(r2, UUID) req = ( select([ZoneTable.c.z_id, DepartmentTable.c.d_id]) .select_from(ZoneTable.join(DepartmentTable).join(MassifTable)) .where(MassifTable.c.m_id == bindparam("massif")) ) id1 = con.execute(req, massif=r1).first() id2 = con.execute(req, massif=r2).first() assert id1.z_id == id2.z_id assert id1.d_id == id2.d_id
def import_flowcapt_station(): db = create_database_connections().engine with connection_scope(db) as con: with resource_stream("nivo_api", "cli/data/flowcapt.geojson") as fp: gj = geojson.load(fp) for station in gj.features: persist_flowcapt_station(con, station)
def import_all_bra(): """ Same as `import_bra` but we request from March 2016 to now. """ start_date = date(year=2016, month=3, day=10) date_range = [ date.today() - timedelta(days=x) for x in range(0, (date.today() - start_date).days + 1) ] for d in date_range: massif = "" try: bra_dates = get_bra_date(d) with connection_scope() as con: for massif, m_date in bra_dates.items(): if not check_bra_record_exist(con, massif, m_date): xml = get_bra_xml(massif, m_date) processed_bra = process_xml(con, xml) persist_bra(con, processed_bra) click.echo(f"Persist {massif.capitalize()}") except Exception as e: log.debug(e) log.critical( f"an error occured when processing massif {massif} for date {d}" )
def find_and_replace_foreign_key_value(self) -> List[Dict]: def replace_num_sta_by_column_name(line: Dict, con: Connection) -> Dict: """ You have to know that some station have no id (yes...) """ nivo_sensor = int(line["nr_nivo_sensor"]) s = select([ SensorStationTable.c.nss_id ]).where(SensorStationTable.c.nss_meteofrance_id == nivo_sensor) res = con.execute(s).first() if res is None: logger.warning( f"No station have been found for id {nivo_sensor} creating an empty one." ) res = create_new_unknown_nivo_sensor_station(nivo_sensor, con) line["nr_nivo_sensor"] = res.nss_id return line with connection_scope() as con: replaced_csv = list() for line in self.cleaned_csv: replaced_csv.append(replace_num_sta_by_column_name(line, con)) self.cleaned_csv = replaced_csv return self.cleaned_csv
def import_nivo_sensor_station(): # this need refactor res = requests.get(f"{Config.METEO_FRANCE_NIVO_BASE_URL}/postesNivo.json") res.raise_for_status() with connection_scope() as con: with con.begin(): for feature in res.json()["features"]: pointz = feature["geometry"] pointz["coordinates"].append( int(feature["properties"]["Altitude"])) mf_id = (feature["properties"]["ID"] if feature["properties"]["ID"] != "" else None) ins = (insert(SensorStationTable).values( **{ "nss_name": feature["properties"]["Nom"], "nss_meteofrance_id": mf_id, "the_geom": func.ST_SetSRID( func.ST_GeomFromGeoJSON(json.dumps(pointz)), 4326), }).on_conflict_do_nothing(index_elements=["nss_name"])) con.execute(ins) inserted = (con.execute( select([func.count(SensorStationTable.c.nss_id).label("count") ])).first().count) click.echo(f"{inserted} sensor station imported")
def _get_bra_by_uuid_or_404(self, bra_id: UUID) -> _Element: with connection_scope() as con: query = select([BraRecordTable.c.br_raw_xml ]).where(BraRecordTable.c.br_id == bra_id) res = con.execute(query).first() if not res: bra_api.abort(HTTPStatus.NOT_FOUND) return res.br_raw_xml
def test_multi_insert(self): with connection_scope() as con: uuid_list = list() for _ in range(5): uuid_list.append(persist_zone(con, "this_is_a_test")) for x in uuid_list: assert isinstance(x, UUID) assert all(x == uuid_list[0] for x in uuid_list)
def test_download_fail(self, database): url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/nivo.20190812.csv" responses.add(responses.GET, url, status=503) with connection_scope(database.engine) as con: with pytest.raises(HTTPError): download_nivo( NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)), con)
def import_nivo(csv_file: ANivoCsv) -> None: csv_file.normalize() csv_file.find_and_replace_foreign_key_value() with connection_scope() as con: with con.begin(): ins = insert(NivoRecordTable).values( csv_file.cleaned_csv ) # .on_conflict_do_nothing(index_elements=['nss_name']) con.execute(ins)
def test_massif(self): with connection_scope() as con: r = persist_massif( con, "CHABLAIS", {"name": "Haute-savoie", "number": "74"}, "Alpes du Nord", ) assert isinstance(r, UUID)
def test_import_nivo(self, database): with open(os.path.join(CURRENT_DIR, "test_data/nivo.20190812.csv")) as f: with connection_scope(database.engine) as con: nivo_csv = DictReader(f, delimiter=";") n = NivoCsv( NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)), con) n.nivo_csv = nivo_csv
def test_create_new_sensor_station_fail(self): """ It should fail when two unknown station have the same name. non-idempotency is assumed (tech debt FTW) """ with connection_scope() as con: with pytest.raises(IntegrityError): r = create_new_unknown_nivo_sensor_station(10, con) assert isinstance(r.nss_id, UUID) create_new_unknown_nivo_sensor_station(10, con)
def check_nivo_doesnt_exist(nivo_date: date) -> bool: with connection_scope() as con: s = exists([ NivoRecordTable.c.nr_date ]).where(cast(NivoRecordTable.c.nr_date, Date) == nivo_date) s = select([s.label("exists")]) does_nivo_already_exist = con.execute(s).first().exists logger.debug( f"does nivo for date {nivo_date.strftime('%d-%m-%Y')} already exist : {does_nivo_already_exist}" ) return not does_nivo_already_exist
def import_last_nivo_data(): # setup # get http://donneespubliques.meteofrance.fr/donnees_libres/Txt/Nivo/lastNivo.js # get last nivo data if needed # process it # import it. db = create_database_connections().engine with connection_scope(db) as con: last_nivo = get_last_nivo_date() if check_nivo_doesnt_exist(con, last_nivo.nivo_date): downloaded_nivo = download_nivo(last_nivo, con) import_nivo(con, downloaded_nivo)
def test_wrong_file_normalize(self, database): file = resource_filename( "test", "test_cli/test_nivo_record_helper/test_data/nivo.wrong_data.csv") with connection_scope(database.engine) as con: date = NivoDate(False, datetime.now()) a = NivoCsv(date, con, "http://test") a.nivo_csv = DictReader(file, delimiter=";") with pytest.raises(KeyError) as e: a.normalize() assert str(e.value) == "'nr_numer_sta'"
def test_archive_download(self, database): url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/Archive/nivo.201701.csv.gz" with open(os.path.join(CURRENT_DIR, "test_data/nivo.201701.csv.gz"), "rb") as f: responses.add(responses.GET, url, body=f.read(), content_type="application/x-gzip") with connection_scope(database.engine) as con: r = download_nivo( NivoDate(is_archive=True, nivo_date=date(2017, 1, 1)), con) assert isinstance(r, ArchiveNivoCss) assert r.nivo_date == date(2017, 1, 1)
def test_recent_nivo_download(self, database): url = f"{Config.METEO_FRANCE_NIVO_BASE_URL}/nivo.20190812.csv" with open(os.path.join(CURRENT_DIR, "test_data/nivo.20190812.csv")) as f: responses.add(responses.GET, url, body=f.read(), content_type="text/plain") with connection_scope(database.engine) as con: r = download_nivo( NivoDate(is_archive=False, nivo_date=date(2019, 8, 12)), con) assert isinstance(r, NivoCsv) assert r.nivo_date == date(2019, 8, 12)
def test_fetch_wrong_file(self, database): responses.add(responses.GET, "http://example.com", content_type="text/html", status=302) date = NivoDate(False, datetime.now()) with connection_scope(database.engine) as con: a = NivoCsv(date, con, "http://example.com") assert a.download_url == "http://example.com" assert a.nivo_date == date.nivo_date with pytest.raises(HTTPError) as ex: a.fetch_and_parse() assert str(ex.value) == "Cannot found Nivo record"
def test_fetch(self, database): with resource_stream( "test", "test_cli/test_nivo_record_helper/test_data/nivo.wrong_data.csv" ) as f: responses.add(responses.GET, "http://test", body=f.read(), content_type="text/plain") with connection_scope(database.engine) as con: date = NivoDate(False, datetime.now()) a = NivoCsv(date, con, "http://test") assert a.download_url == "http://test" assert a.nivo_date == date.nivo_date
def test_expected_json(database): test_f = Feature( properties={ "id": "FBER1", "site": "La Berarde", "altitude": 2390, "country": "France", }, geometry={"type": "Point", "coordinates": [6.237082, 44.949944]}, ) with connection_scope(database.engine) as con: persist_flowcapt_station(con, test_f) res = con.execute(select([FlowCaptStationTable])).fetchall() assert len(res) == 1 assert res[0].fcs_id == "FBER1"
def import_last_bra(): # Open data from meteo france via it's obscure rest api because they are not providing opendata for the current day # They start providing the BRA one day after it's been released, which makes the app useless. with connection_scope() as con: for dept in DepartmentTable.get(con): try: dept = get_bra_by_dept_from_mf_rpc_api(dept.d_number) for massif in dept: xml = format_xml_from_mf_rpc(massif["corpsBulletin"]) processed_bra = process_xml(con, xml) persist_bra(con, processed_bra) except HTTPError as e: log.critical(f"dept {dept.d_name} cannot be fetch no BRA") continue except Exception as e: log.critical( f"an error occured when processing dept {dept.d_name} for today" )
def _inject_test_data(self): with connection_scope() as con: nss_id = uuid4() con.execute(SensorStationTable.insert().values({ "nss_id": nss_id, "nss_name": "test", "nss_meteofrance_id": 1, "the_geom": "SRID=4326;POINT(1 1 1)", })) con.execute(NivoRecordTable.insert().values({ "nr_date": date(2019, 1, 1), "nr_nivo_sensor": nss_id }))
def import_massifs(): massif_json = requests.get(Config.BRA_BASE_URL + "/massifs.json").json() with connection_scope() as con: # the 4th element of the massif is useless, and there are no BRA for it. for zone in massif_json[:4]: for dept in zone["departements"]: for massif in dept["massifs"]: click.echo(f"Importing {massif}") try: persist_massif( con, massif, { "name": dept["nom_dep"], "number": dept["num_dep"] }, zone["zone"], ) except ValueError: log.warning(f"Do no import massif: {massif}")
def import_all_nivo_data(): # setup # download from 2010 to now # download all file (via http, better) # process it # import it all_nivo_date = get_all_nivo_date() log.info(f"Need to process {len(all_nivo_date)}") db = create_database_connections().engine with connection_scope(db) as con: for nivo_date in all_nivo_date: if check_nivo_doesnt_exist(con, nivo_date.nivo_date): try: log.info( f"Processing for {nivo_date.nivo_date.strftime('%d-%m-%Y')}" ) downloaded_nivo = download_nivo(nivo_date, con) import_nivo(con, downloaded_nivo) except Exception as e: click.echo("Something bad append") log.debug(e)
def import_last_bra(): click.echo("This function doesn't work anymore sorry") sys.exit() # This is obsolete. db = create_database_connections().engine with connection_scope(db) as con: for dept in DepartmentTable.get(con): try: dept = get_bra_by_dept_from_mf_rpc_api(dept.d_number) for massif in dept: xml = format_xml_from_mf_rpc(massif["corpsBulletin"]) processed_bra = process_xml(con, xml) persist_bra(con, processed_bra) except HTTPError as e: log.critical(f"dept {dept['d_name']} cannot be fetch no BRA") log.debug(e) continue except Exception as e: log.critical( f"an error occured when processing dept {dept['d_name']} for today" ) log.debug(e)
def import_bra(bra_date): """ * setup * request https://donneespubliques.meteofrance.fr/donnees_libres/Pdf/BRA/bra.%Y%m%d.json with all the date from december 2016 to today * if not 302 (302 means 404 at meteofrance ðŸ˜) * for all the date in all the json, download the xml of bra * process (download + post process) * import """ bra_dates = get_bra_date(bra_date) with connection_scope() as con: for massif, m_date in bra_dates.items(): try: if not check_bra_record_exist(con, massif, m_date): xml = get_bra_xml(massif, m_date) processed_bra = process_xml(con, xml) persist_bra(con, processed_bra) click.echo(f"Persist {massif.capitalize()}") except Exception as e: log.debug(e) log.critical( f"an error occured when processing massif {massif} for date {m_date}" )
def test_insert_zone(self): with connection_scope() as con: r = persist_zone(con, "this_is_a_test") assert isinstance(r, UUID)
def test_no_massif(self): with connection_scope() as con: with pytest.raises(ValueError) as e: _get_massif_id("CHABLAIS", con) assert e.value.args[0] == "Cannot found massif CHABLAIS in the db" assert e.type is ValueError
def test_process_xml_work(self, bra_xml_parsed): self._setup_test() with connection_scope() as c: p = process_xml(c, bra_xml_parsed) assert isinstance(p, list)