def get_observations_for_web(info_role): """ Optimized route for serve data to the frontend with all filters Parameters: - GET : all the fields of the view v_synthese_for_export Return: Array of dict (with geojson key) """ filters = { key: request.args.getlist(key) for key, value in request.args.items() } if "limit" in filters: result_limit = filters.pop("limit")[0] else: result_limit = current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"] query = (select([ VSyntheseForWebApp.id_synthese, VSyntheseForWebApp.date_min, VSyntheseForWebApp.lb_nom, VSyntheseForWebApp.cd_nom, VSyntheseForWebApp.nom_vern, VSyntheseForWebApp.st_asgeojson, VSyntheseForWebApp.observers, VSyntheseForWebApp.dataset_name, VSyntheseForWebApp.url_source, VSyntheseForWebApp.entity_source_pk_value, ]).where(VSyntheseForWebApp.the_geom_4326.isnot(None)).order_by( VSyntheseForWebApp.date_min.desc())) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, query, filters) synthese_query_class.filter_query_all_filters(info_role) result = DB.engine.execute(synthese_query_class.query.limit(result_limit)) geojson_features = [] for r in result: properties = { "id": r["id_synthese"], "date_min": str(r["date_min"]), "cd_nom": r["cd_nom"], "nom_vern_or_lb_nom": r["nom_vern"] if r["nom_vern"] else r["lb_nom"], "lb_nom": r["lb_nom"], "dataset_name": r["dataset_name"], "observers": r["observers"], "url_source": r["url_source"], "entity_source_pk_value": r["entity_source_pk_value"], } geojson = ast.literal_eval(r["st_asgeojson"]) geojson["properties"] = properties geojson_features.append(geojson) return { "data": FeatureCollection(geojson_features), "nb_total": len(geojson_features), "nb_obs_limited": len(geojson_features) == current_app.config["SYNTHESE"] ["NB_MAX_OBS_MAP"], }
def get_observations_for_web(info_role): """ Optimized route for serve data to the frontend with all filters Parameters: - GET : all the fields of the view v_synthese_for_export Return: Array of dict (with geojson key) """ filters = {key: request.args.getlist(key) for key, value in request.args.items()} if "limit" in filters: result_limit = filters.pop("limit")[0] else: result_limit = current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"] query = ( select( [ VSyntheseForWebApp.id_synthese, VSyntheseForWebApp.date_min, VSyntheseForWebApp.lb_nom, VSyntheseForWebApp.cd_nom, VSyntheseForWebApp.nom_vern, VSyntheseForWebApp.st_asgeojson, VSyntheseForWebApp.observers, VSyntheseForWebApp.dataset_name, VSyntheseForWebApp.url_source, VSyntheseForWebApp.entity_source_pk_value, ] ) .where(VSyntheseForWebApp.the_geom_4326.isnot(None)) .order_by(VSyntheseForWebApp.date_min.desc()) ) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, query, filters) synthese_query_class.filter_query_all_filters(info_role) result = DB.engine.execute(synthese_query_class.query.limit(result_limit)) geojson_features = [] for r in result: properties = { "id": r["id_synthese"], "date_min": str(r["date_min"]), "cd_nom": r["cd_nom"], "nom_vern_or_lb_nom": r["nom_vern"] if r["nom_vern"] else r["lb_nom"], "lb_nom": r["lb_nom"], "dataset_name": r["dataset_name"], "observers": r["observers"], "url_source": r["url_source"], "entity_source_pk_value": r["entity_source_pk_value"], } geojson = ast.literal_eval(r["st_asgeojson"]) geojson["properties"] = properties geojson_features.append(geojson) return { "data": FeatureCollection(geojson_features), "nb_total": len(geojson_features), "nb_obs_limited": len(geojson_features) == current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"], }
def export_metadata(info_role): """Route to export the metadata in CSV .. :quickref: Synthese; The table synthese is join with gn_synthese.v_metadata_for_export The column jdd_id is mandatory in the view gn_synthese.v_metadata_for_export POST parameters: Use a list of id_synthese (in POST parameters) to filter the v_synthese_for_export_view """ if request.json: filters = request.json elif request.data: # decode byte to str - compat python 3.5 filters = json.loads(request.data.decode("utf-8")) else: filters = { key: request.args.getlist(key) for key, value in request.args.items() } metadata_view = GenericTable(tableName="v_metadata_for_export", schemaName="gn_synthese", engine=DB.engine) q = DB.session.query(distinct(VSyntheseForWebApp.id_dataset), metadata_view.tableDef).join( metadata_view.tableDef, getattr( metadata_view.tableDef.columns, current_app.config["SYNTHESE"] ["EXPORT_METADATA_ID_DATASET_COL"], ) == VSyntheseForWebApp.id_dataset, ) q = select( [distinct(VSyntheseForWebApp.id_dataset), metadata_view.tableDef]) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, q, filters) synthese_query_class.add_join( metadata_view.tableDef, getattr( metadata_view.tableDef.columns, current_app.config["SYNTHESE"]["EXPORT_METADATA_ID_DATASET_COL"], ), VSyntheseForWebApp.id_dataset) synthese_query_class.filter_query_all_filters(info_role) data = DB.engine.execute(synthese_query_class.query) return to_csv_resp( datetime.datetime.now().strftime("%Y_%m_%d_%Hh%Mm%S"), data=[metadata_view.as_dict(d) for d in data], separator=";", columns=[db_col.key for db_col in metadata_view.tableDef.columns], )
def get_synthese(info_role): """Return synthese row(s) filtered by form params. NOT USED ANY MORE FOR PERFORMANCE ISSUES .. :quickref: Synthese; Deprecated .. deprecated:: 2? Use :route: /for_web instead Params must have same synthese fields names :parameter str info_role: Role used to get the associated filters :returns dict[dict, int, bool]: See description above """ # change all args in a list of value filters = { key: request.args.getlist(key) for key, value in request.args.items() } if "limit" in filters: result_limit = filters.pop("limit")[0] else: result_limit = current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"] query = select([VSyntheseForWebApp ]).order_by(VSyntheseForWebApp.date_min.desc()) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, query, filters) synthese_query_class.filter_query_all_filters(info_role) data = DB.engine.execute(synthese_query_class.query.limit(result_limit)) # q = synthese_query.filter_query_all_filters(VSyntheseForWebApp, q, filters, info_role) # data = q.limit(result_limit) columns = current_app.config["SYNTHESE"][ "COLUMNS_API_SYNTHESE_WEB_APP"] + MANDATORY_COLUMNS features = [] for d in data: feature = d.get_geofeature(columns=columns) feature["properties"]["nom_vern_or_lb_nom"] = ( d.lb_nom if d.nom_vern is None else d.nom_vern) features.append(feature) return { "data": FeatureCollection(features), "nb_obs_limited": len(features) == current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"], "nb_total": len(features), }
def export_status(info_role): """ Route to get all the protection status of a synthese search Parameters: - HTTP-GET: the same that the /synthese endpoint (all the filter in web app) Get the CRUVED from 'R' action because we don't give observations X/Y but only statuts and to be constistant with the data displayed in the web interface """ filters = {key: request.args.getlist(key) for key, value in request.args.items()} # initalize the select object q = select( [ distinct(VSyntheseForWebApp.cd_nom), Taxref.nom_complet, Taxref.cd_ref, Taxref.nom_vern, TaxrefProtectionArticles.type_protection, TaxrefProtectionArticles.article, TaxrefProtectionArticles.intitule, TaxrefProtectionArticles.arrete, TaxrefProtectionArticles.date_arrete, TaxrefProtectionArticles.url, ] ) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, q, filters) # add join synthese_query_class.add_join(Taxref, Taxref.cd_nom, VSyntheseForWebApp.cd_nom) synthese_query_class.add_join( TaxrefProtectionEspeces, TaxrefProtectionEspeces.cd_nom, VSyntheseForWebApp.cd_nom, ) synthese_query_class.add_join( TaxrefProtectionArticles, TaxrefProtectionArticles.cd_protection, TaxrefProtectionEspeces.cd_protection, ) # filter with all get params q = synthese_query_class.filter_query_all_filters(info_role) data = DB.engine.execute(q) protection_status = [] for d in data: row = OrderedDict( [ ("nom_complet", d["nom_complet"]), ("nom_vern", d["nom_vern"]), ("cd_nom", d["cd_nom"]), ("cd_ref", d["cd_ref"]), ("type_protection", d["type_protection"]), ("article", d["article"]), ("intitule", d["intitule"]), ("arrete", d["arrete"]), ("date_arrete", d["date_arrete"]), ("url", d["url"]), ] ) protection_status.append(row) export_columns = [ "nom_complet", "nom_vern", "cd_nom", "cd_ref", "type_protection", "article", "intitule", "arrete", "date_arrete", "url", ] return to_csv_resp( datetime.datetime.now().strftime("%Y_%m_%d_%Hh%Mm%S"), protection_status, separator=";", columns=export_columns, )
def get_synthese_data(info_role): """ Return synthese and t_validations data filtered by form params Params must have same synthese fields names .. :quickref: Validation; Parameters: ------------ info_role (User): Information about the user asking the route. Auto add with kwargs Returns ------- dict { "data": FeatureCollection "nb_obs_limited": int est-ce que le nombre de données retournée est > au nb limites "nb_total": nb_total, } """ if request.json: filters = request.json elif request.data: # decode byte to str - compat python 3.5 filters = json.loads(request.data.decode("utf-8")) else: filters = { key: request.args.get(key) for key, value in request.args.items() } if "limit" in filters: result_limit = filters.pop("limit") else: result_limit = blueprint.config["NB_MAX_OBS_MAP"] # Construction de la requête select # Les champs correspondent aux champs obligatoires # + champs définis par l'utilisateur columns = (blueprint.config["COLUMNS_API_VALIDATION_WEB_APP"] + blueprint.config["MANDATORY_COLUMNS"]) select_columns = [] serializer = {} for c in columns: try: att = getattr(VSyntheseValidation, c) select_columns.append(att) serializer[c] = SERIALIZERS.get( att.type.__class__.__name__.lower(), lambda x: x) except AttributeError as error: log.warning("Validation : colonne {} inexistante".format(c)) # Construction de la requête avec SyntheseQuery # Pour profiter des opérations CRUVED query = (select(select_columns).where( VSyntheseValidation.the_geom_4326.isnot(None)).order_by( VSyntheseValidation.date_min.desc())) validation_query_class = SyntheseQuery(VSyntheseValidation, query, filters) validation_query_class.filter_query_all_filters(info_role) result = DB.engine.execute( validation_query_class.query.limit(result_limit)) # TODO nb_total factice nb_total = 0 geojson_features = [] properties = {} for r in result: properties = {k: serializer[k](r[k]) for k in serializer.keys()} properties["nom_vern_or_lb_nom"] = (r["nom_vern"] if r["nom_vern"] else r["lb_nom"]) geojson = ast.literal_eval(r["geojson"]) geojson["properties"] = properties geojson["id"] = r["id_synthese"] geojson_features.append(geojson) return { "data": FeatureCollection(geojson_features), "nb_obs_limited": nb_total == blueprint.config["NB_MAX_OBS_MAP"], "nb_total": nb_total, }
def get_synthese_data(info_role): """ Return synthese and t_validations data filtered by form params Params must have same synthese fields names .. :quickref: Validation; Parameters: ------------ info_role (User): Information about the user asking the route. Auto add with kwargs truc (int): essai Returns ------- dict test """ # try: filters = { key: request.args.getlist(key) for key, value in request.args.items() } for key, value in filters.items(): if "," in value[0] and key != "geoIntersection": filters[key] = value[0].split(",") if "limit" in filters: result_limit = filters.pop("limit")[0] else: result_limit = blueprint.config["NB_MAX_OBS_MAP"] query = (select([ VSyntheseValidation.cd_nomenclature_validation_status, VSyntheseValidation.dataset_name, VSyntheseValidation.date_min, VSyntheseValidation.id_nomenclature_valid_status, VSyntheseValidation.id_synthese, VSyntheseValidation.nom_valide, VSyntheseValidation.nom_vern, VSyntheseValidation.geojson, VSyntheseValidation.observers, VSyntheseValidation.validation_auto, VSyntheseValidation.validation_date, VSyntheseValidation.nom_vern, VSyntheseValidation.lb_nom, VSyntheseValidation.cd_nom, VSyntheseValidation.comment_description, VSyntheseValidation.altitude_min, VSyntheseValidation.altitude_max, VSyntheseValidation.unique_id_sinp, VSyntheseValidation.meta_update_date, ]).where(VSyntheseValidation.the_geom_4326.isnot(None)).order_by( VSyntheseValidation.date_min.desc())) validation_query_class = SyntheseQuery(VSyntheseValidation, query, filters) validation_query_class.filter_query_all_filters(info_role) result = DB.engine.execute( validation_query_class.query.limit(result_limit)) nb_total = 0 columns = (blueprint.config["COLUMNS_API_VALIDATION_WEB_APP"] + blueprint.config["MANDATORY_COLUMNS"]) geojson_features = [] for r in result: properties = { "id_synthese": r["id_synthese"], "cd_nomenclature_validation_status": r["cd_nomenclature_validation_status"], "dataset_name": r["dataset_name"], "date_min": str(r["date_min"]), "nom_vern_or_lb_nom": r["nom_vern"] if r["nom_vern"] else r["lb_nom"], "observers": r["observers"], "validation_auto": r["validation_auto"], "validation_date": str(r["validation_date"]), "altitude_min": r["altitude_min"], "altitude_max": r["altitude_max"], "comment": r["comment_description"], "cd_nom": r["cd_nom"], "unique_id_sinp": str(r["unique_id_sinp"]), "meta_update_date": str(r["meta_update_date"]), } geojson = ast.literal_eval(r["geojson"]) geojson["properties"] = properties geojson["id"] = r["id_synthese"] geojson_features.append(geojson) return { "data": FeatureCollection(geojson_features), "nb_obs_limited": nb_total == blueprint.config["NB_MAX_OBS_MAP"], "nb_total": nb_total, }
def get_observations_for_web(info_role): """Optimized route to serve data for the frontend with all filters. .. :quickref: Synthese; Get filtered observations Query filtered by any filter, returning all the fields of the view v_synthese_for_export:: properties = { "id": r["id_synthese"], "date_min": str(r["date_min"]), "cd_nom": r["cd_nom"], "nom_vern_or_lb_nom": r["nom_vern"] if r["nom_vern"] else r["lb_nom"], "lb_nom": r["lb_nom"], "dataset_name": r["dataset_name"], "observers": r["observers"], "url_source": r["url_source"], "entity_source_pk_value": r["entity_source_pk_value"], } geojson = ast.literal_eval(r["st_asgeojson"]) geojson["properties"] = properties :param str info_role: Role used to get the associated filters, **TBC** :qparam str limit: Limit number of synthese returned. Defaults to NB_MAX_OBS_MAP. :qparam str cd_ref: Filter by TAXREF cd_ref attribute :qparam str taxonomy_group2_inpn: Filter by TAXREF group2_inpn attribute :qparam str taxonomy_id_hab: Filter by TAXREF id_habitat attribute :qparam str taxonomy_lr: Filter by TAXREF cd_ref attribute :qparam str taxhub_attribut*: Generig TAXREF filter, given attribute & value :qparam str observers: Filter on observer :qparam str id_organism: Filter on organism :qparam str date_min: Start date :qparam str date_max: End date :qparam str id_acquisition_framework: *tbd* :qparam str geoIntersection: Intersect with the geom send from the map :qparam str period_start: *tbd* :qparam str period_end: *tbd* :qparam str area*: Generic filter on area :qparam str *: Generic filter, given by colname & value :>jsonarr array data: Array of synthese with geojson key, see above :>jsonarr int nb_total: Number of observations :>jsonarr bool nb_obs_limited: Is number of observations capped """ if request.json: filters = request.json elif request.data: # decode byte to str - compat python 3.5 filters = json.loads(request.data.decode("utf-8")) else: filters = { key: request.args.getlist(key) for key, value in request.args.items() } # Passage de l'ensemble des filtres # en array pour des questions de compatibilité # TODO voir si ça ne peut pas être modifié for k in filters.keys(): if not isinstance(filters[k], list): filters[k] = [filters[k]] if "limit" in filters: result_limit = filters.pop("limit")[0] else: result_limit = current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"] query = ( select( [ VSyntheseForWebApp.id_synthese, VSyntheseForWebApp.date_min, VSyntheseForWebApp.lb_nom, VSyntheseForWebApp.cd_nom, VSyntheseForWebApp.nom_vern, VSyntheseForWebApp.st_asgeojson, VSyntheseForWebApp.observers, VSyntheseForWebApp.dataset_name, VSyntheseForWebApp.url_source, VSyntheseForWebApp.entity_source_pk_value, ] ) .where(VSyntheseForWebApp.the_geom_4326.isnot(None)) .order_by(VSyntheseForWebApp.date_min.desc()) ) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, query, filters) synthese_query_class.filter_query_all_filters(info_role) result = DB.engine.execute(synthese_query_class.query.limit(result_limit)) geojson_features = [] for r in result: properties = { "id": r["id_synthese"], "date_min": str(r["date_min"]), "cd_nom": r["cd_nom"], "nom_vern_or_lb_nom": r["nom_vern"] if r["nom_vern"] else r["lb_nom"], "lb_nom": r["lb_nom"], "dataset_name": r["dataset_name"], "observers": r["observers"], "url_source": r["url_source"], "entity_source_pk_value": r["entity_source_pk_value"], } geojson = ast.literal_eval(r["st_asgeojson"]) geojson["properties"] = properties geojson_features.append(geojson) return { "data": FeatureCollection(geojson_features), "nb_total": len(geojson_features), "nb_obs_limited": len(geojson_features) == current_app.config["SYNTHESE"]["NB_MAX_OBS_MAP"], }
def export_status(info_role): """Route to get all the protection status of a synthese search .. :quickref: Synthese; Get the CRUVED from 'R' action because we don't give observations X/Y but only statuts and to be constistant with the data displayed in the web interface Parameters: - HTTP-GET: the same that the /synthese endpoint (all the filter in web app) """ filters = {key: request.args.getlist(key) for key, value in request.args.items()} # initalize the select object q = select( [ distinct(VSyntheseForWebApp.cd_nom), Taxref.nom_complet, Taxref.cd_ref, Taxref.nom_vern, TaxrefProtectionArticles.type_protection, TaxrefProtectionArticles.article, TaxrefProtectionArticles.intitule, TaxrefProtectionArticles.arrete, TaxrefProtectionArticles.date_arrete, TaxrefProtectionArticles.url, ] ) synthese_query_class = SyntheseQuery(VSyntheseForWebApp, q, filters) # add join synthese_query_class.add_join( Taxref, Taxref.cd_nom, VSyntheseForWebApp.cd_nom) synthese_query_class.add_join( TaxrefProtectionEspeces, TaxrefProtectionEspeces.cd_nom, VSyntheseForWebApp.cd_nom, ) synthese_query_class.add_join( TaxrefProtectionArticles, TaxrefProtectionArticles.cd_protection, TaxrefProtectionEspeces.cd_protection, ) # filter with all get params q = synthese_query_class.filter_query_all_filters(info_role) data = DB.engine.execute(q) protection_status = [] for d in data: row = OrderedDict( [ ("nom_complet", d["nom_complet"]), ("nom_vern", d["nom_vern"]), ("cd_nom", d["cd_nom"]), ("cd_ref", d["cd_ref"]), ("type_protection", d["type_protection"]), ("article", d["article"]), ("intitule", d["intitule"]), ("arrete", d["arrete"]), ("date_arrete", d["date_arrete"]), ("url", d["url"]), ] ) protection_status.append(row) export_columns = [ "nom_complet", "nom_vern", "cd_nom", "cd_ref", "type_protection", "article", "intitule", "arrete", "date_arrete", "url", ] return to_csv_resp( datetime.datetime.now().strftime("%Y_%m_%d_%Hh%Mm%S"), protection_status, separator=";", columns=export_columns, )