def pysdk_checking(self):
logger.debug("\n--------------- isogeo-pysdk ---------------")
logger.debug("Checking credentials")
try:
isogeo = Isogeo(self.app_id, self.app_secrets)
isogeo.connect()
except OSError as e:
logger.debug("Credentials issue : {}".format(e))
return
except ValueError as e:
logger.debug("Credentials issue : {}".format(e))
return
if self.search_type == "init":
result = isogeo.search(
whole_share=0, page_size=0, augment=0, tags_as_dicts=1
)
else:
query = " ".join(list(self.checked_kw.keys()))
result = isogeo.search(
whole_share=0, page_size=0, augment=0, tags_as_dicts=1, query=query
)
self.tags_expected = result.get("tags")
self.kw_expected_nb = len(self.tags_expected.get("keywords"))
self.ui.lbl_expected.setText(
"Expected : {} resources and {} keywords".format(
result.get("total"), self.kw_expected_nb
)
)
logger.debug(
"isogeo-pysdk validates the authentication file, {} accessible resources".format(
result.get("total")
)
)
class IsogeoAPI(): # Une classe qui hérite de la classe Isogeo
""" Make easier connecting and requesting to Isogeo's API using isogeo-pysdk package.
Online resources:
* Full isogeo-pysdk doc at : https://isogeo-api-pysdk.readthedocs.io/en/latest/index.html
* Full Isogeo's API doc at : http://help.isogeo.com/api
:param str file_name: name of the credential file
"""
def __init__(self, file_name: str = "client_secrets.json") -> object:
# création du chemin vers le fichier d'authentification
self.json_file = "{}/{}".format("/".join(__file__.split("\\")[:-1]),
file_name)
# récupérétion des informations d'authentification contenues dans le fichier
self.client_id = utils.credentials_loader(
self.json_file).get("client_id")
self.client_secret = utils.credentials_loader(
self.json_file).get("client_secret")
# connexion à l'API et récupération du token
self.isogeo = Isogeo(
client_id=self.client_id,
client_secret=self.client_secret,
auth_mode="group",
auto_refresh_url="https://id.api.isogeo.com/oauth/token",
platform="prod")
self.isogeo.connect()
# Méthode permettant d'effectuer les différents types de requête à l'API
def request_maker(self,
filter_request: bool = 0,
filter_query: str = None) -> dict:
"""Request API about the resources shared with the application wich credential file allow access.
Using a method of isogeo_pysdk's Isogeo class to return the number and the tags of searched resources.
:param bool filter_request: 0 [DEFAULT] to search all the resources shared in the application / 1 to filter the request
:param str filter_query: with filter_request = 1, character string for 'query' parameter of * isogeo_pysdk.Isogeo.search() * method
"""
if filter_request == 1:
search = self.isogeo.search(page_size=0,
augment=0,
tags_as_dicts=1,
query=filter_query)
else:
search = self.isogeo.search(page_size=0,
augment=0,
tags_as_dicts=1)
# retourne les valeurs des champs ainsi que le nombre de métadonnées filtrées
return search.tags, search.total
class TestIsogeo(unittest.TestCase):
# ------------ Basic methods ---------------------------------------------
def setUp(self):
"""Before"""
# reading config file
settings_file = "../isogeo_pysdk/isogeo_params.ini"
if not path.isfile(settings_file):
return
else:
pass
config = ConfigParser.SafeConfigParser()
config.read(settings_file)
self.share_id = config.get('auth', 'app_id')
self.share_token = config.get('auth', 'app_secret')
self.isogeo = Isogeo(client_id=self.share_id,
client_secret=self.share_token)
self.jeton = self.isogeo.connect()
def tearDown(self):
"""After"""
# print("Cleaned up!")
# ------------ Tests methods ---------------------------------------------
def test_authentication(self):
"""Authentication and connection to Isogeo API"""
# jeton = isogeo.connect()
def test_connection(self):
"""Authentication and connection to Isogeo API"""
self.jeton = self.isogeo.connect()
def test_search(self):
"""Isogeo API search"""
search_empty = self.isogeo.search(self.jeton)
assert(type(search_empty) != unicode)
assert(type(search_empty) == dict)
assert("envelope" in search_empty.keys())
assert("limit" in search_empty.keys())
assert("offset" in search_empty.keys())
assert("query" in search_empty.keys())
assert("results" in search_empty.keys())
assert("tags" in search_empty.keys())
assert("total" in search_empty.keys())
assert(type(search_empty.get("results")) == list)
def _update_db(self):
"""Update metadata list from API."""
# get stored metadata
db_mds = Metadata.objects.all()
db_wgs = Workgroup.objects.all()
# connect to isogeo
isogeo = Isogeo(client_id=settings.ISOGEO_CLIENT_ID,
client_secret=settings.ISOGEO_CLIENT_SECRET,
lang="fr")
token = isogeo.connect()
search = isogeo.search(token,
# page_size=10,
order_by="modified",
# whole_share=0,
# sub_resources=["events"]
)
# tags
tags = search.get("tags")
for tag in tags:
if tag.startswith("owner"):
new_owner = Workgroup(isogeo_uuid=tag[6:-1],
label=tags.get(tag))
new_owner.save()
# metadatas
# for md in search.get("results"):
# try:
# new_md = Metadata(isogeo_id=md.get("_id"),
# title=md.get("title", "No title"),
# name=md.get("name"),
# abstract=md.get("abstract"),
# md_dt_crea=md.get("_created"),
# md_dt_update=md.get("_modified"),
# rs_dt_crea=md.get("created"),
# rs_dt_update=md.get("modified"),
# source=True)
# new_md.save()
# logging.info("Metadata added")
# except IntegrityError:
# # in case of duplicated offer
# logging.error("Metadata already existed")
# continue
logging.info("{} metadata added")
def pysdk_checking(self):
logger.debug("\n------------------ isogeo-pysdk ------------------")
logger.debug("Checking credentials")
try:
isogeo = Isogeo(self.app_id, self.app_secrets)
isogeo.connect()
except OSError as e:
logger.debug("Credentials issue : {}".format(e))
return
except ValueError as e:
logger.debug("Credentials issue : {}".format(e))
return
self.md_expected = isogeo.search(whole_share=0, page_size=0,
augment=0).get("total")
self.ui.lbl_expected.setText("{} expected resources".format(
self.md_expected))
logger.debug(
"isogeo-pysdk validates the authentication file, {} accessible resources"
.format(self.md_expected))
api_credentials = utils.credentials_loader("client_secrets.json")
isogeo = Isogeo(
client_id=api_credentials.get("client_id"),
client_secret=api_credentials.get("client_secret"),
lang=language,
)
token = isogeo.connect()
# Get basic information
isogeo.get_app_properties(token)
app_props = isogeo.app_properties
print(app_props)
# Get tags to populate filters
search = isogeo.search(token, page_size=0, whole_share=0, augment=1, tags_as_dicts=1)
tags = search.get("tags")
# In case of a need of translation
tr = IsogeoTranslator(language)
# instanciate main window
window = tk.Tk()
window.resizable(width=True, height=True)
window.title("Isogeo Python SDK v{} - Sample desktop search form".format(pysdk_version))
# styling
style = ttk.Style()
s1 = tk.PhotoImage("search1", data=data, format="gif -index 0")
s2 = tk.PhotoImage("search2", data=data, format="gif -index 1")
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
logger.info(
"------------------------- READING EXCEL FILE ----------------------------"
)
xlsx_reader = IsogeoFromxlsx(environ.get("EXCEL_FILE_PATH"))
xlsx_reader.read_file()
xlsx_md_uuids = [record.get("md")._id for record in xlsx_reader.md_read]
search = isogeo.search(include="all",
whole_results=True,
specific_md=tuple(xlsx_md_uuids))
dict_cat_kw = {
"test_1": "d8d24a8306f44cc093c3522c2219d33a",
"test_2": "41245bbed2ad493cb01f5903d1757eeb",
}
if xlsx_reader.lang == "fr":
dict_inspireTh = dict_inspire_fr
isogeo_tr = IsogeoTranslator("FR")
else:
dict_inspireTh = dict_inspire_en
isogeo_tr = IsogeoTranslator("EN")
logger.info(
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
# request Isogeo API about source metadatas
src_cat_search = isogeo.search(
group=origin_wg_uuid,
query="catalog:{}".format(src_cat_uuid),
whole_results=True
)
print("{} source metadatas retrieved".format(src_cat_search.total))
# request Isogeo API about target metadatas
trg_cat_search = isogeo.search(
group=origin_wg_uuid,
query="catalog:{}".format(trg_cat_uuid),
whole_results=True
)
print("{} target metadatas retrieved".format(trg_cat_search.total))
isogeo.close()
# retrieve source metadatas infos from Isogeo API response
"du ": "de ",
"le ": "",
}
sr_mng = SearchReplaceManager(
api_client=isogeo,
attributes_patterns=replace_patterns,
prepositions=dict_prepositions,
)
search_params = {
"group": environ.get("ISOGEO_ORIGIN_WORKGROUP"),
"whole_results": True,
}
# logger.info("search metadatas into client work group")
wg_md_search = isogeo.search(**search_params)
# Keeping only metadata matching with replace pattern
logger.info("making the list of metadatas id matching the pattern")
tup_to_backup = sr_mng.filter_matching_metadatas(wg_md_search.results)
li_to_backup = [md._id for md in tup_to_backup]
# ############ BACKUP #############
logger.info("BACKUP {} METADATAS ".format(len(li_to_backup)))
# instanciate backup manager
backup_path = Path(r"./scripts/dijon/search_and_replace/_output/_backup")
backup_mng = BackupManager(api_client=isogeo, output_folder=backup_path)
# lauching backup
amplitude = 50
bound_range = int(len(li_to_backup) / amplitude)
li_bound = []
# ------------ Real start ----------------
# instanciating Isogeo class
isogeo = Isogeo(client_id=share_id,
client_secret=share_token,
lang="fr")
token = isogeo.connect()
# Windows 10 notifications class
notif = ToastNotifier()
# ------------ REAL START ----------------------------
latest_data_modified = isogeo.search(token,
page_size=10,
order_by="modified",
whole_share=0,
sub_resources=["events"]
)
# parsing the previous date
last_exe = dtparse(last_exe).strftime("%a %d %B %Y (%H:%M)")
# last_exe = last_exe.decode("Latin1")
# comparing total of metadats shared since last time
now_total = latest_data_modified.get('total')
if now_total > last_total:
notif.show_toast("Isogeo - Total partage",
"{} données ajoutées\ndepuis le {}"
.format(now_total - last_total, last_exe),
icon_path=r"img/favicon.ico",
logger.info("Authenticating to Isogeo API")
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
search = isogeo.search(
group=wg_uuid,
whole_results=True,
)
# Build the list of metadata to delete UUID excluding some sheet if needed
li_dlt_uuid = []
nb_excluded = 0
for md in search.results:
if md.get("_id") not in li_excluded_md_uuid and md.get(
"type") != "resource" and md.get("_creator").get(
"_id") == wg_uuid:
li_dlt_uuid.append(md.get("_id"))
else:
nb_excluded += 1
logger.info("excluding '{}' ({}) {} metadata sheet.".format(
md.get("title"), md.get("_id"), md.get("type")))
pass
# API client instanciation
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
# request Isogeo API about source metadatas
whole_search = isogeo.search(
group=origin_wg_uuid,
whole_results=True
)
isogeo.close()
# retrieve source metadatas infos from Isogeo API response
li_md_infos = []
for md in whole_search.results:
li_md_infos.append((md.get("_id"), md.get("title"), md.get("name", "NR")))
# ############################### LOADING ORIGINAL MATCHING TABLE ###############################
input_csv = Path(r"./scripts/belfort/csv/correspondances_light.csv")
fieldnames = [
"source_uuid",
"source_title",
"source_name",
share_id = config.get('auth', 'app_id')
share_token = config.get('auth', 'app_secret')
# ------------ Real start ----------------
# instanciating the class
isogeo = Isogeo(client_id=share_id,
client_secret=share_token)
# getting a token
jeton = isogeo.connect()
# let's search for metadatas!
search = isogeo.search(jeton,
query="owner:b81e0b3bc3124deeadbf59ad05c71a2a",
page_size=10,
whole_share=0,
sub_resources=["layers", "links",
"operations", "serviceLayers"])
# ------------ Parsing resources ----------------
md_resources = OrderedDict()
kind_ogc = ('wfs', 'wms', 'wmts')
kind_esri = ('esriFeatureService', 'esriMapService', 'esriTileService')
li_ogc_share = []
li_esri_share = []
li_dl_share = []
for md in search.get('results'):
if md.get("type") == "service":
print("Services metadatas are excluded.")
# API client instanciation
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_time = default_timer()
new_md = Metadata()
new_md.type = type_code
new_md.format = format_code
new_md.title = title
new_md.name = name
new_md.path = path
metadata = isogeo.metadata.create(workgroup_id=trg_wg_uuid, metadata=new_md)
for cat_uuid in li_trg_cat_uuid:
cat = isogeo.catalog.get(workgroup_id=trg_wg_uuid, catalog_id=cat_uuid)
isogeo.catalog.associate_metadata(metadata=metadata, catalog=cat)
search = isogeo.search()
isogeo.close()
import sys
sys.exit()
else:
pass
# reading ini file
config = ConfigParser.SafeConfigParser()
config.read(settings_file)
share_id = config.get("auth", "app_id")
share_token = config.get("auth", "app_secret")
# ------------ Real start ----------------
# instanciating the class
isogeo = Isogeo(client_id=share_id, client_secret=share_token, lang="fr")
token = isogeo.connect()
# ------------ REAL START ----------------------------
latest_data_modified = isogeo.search(token, page_size=10, order_by="modified", whole_share=0)
for md in latest_data_modified.get("results"):
title = md.get("title")
xml_stream = isogeo.xml19139(token, md.get("_id"))
with open("{}.xml".format(title), "wb") as fd:
for block in xml_stream.iter_content(1024):
fd.write(block)
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
logger.info(
"Retrieving metadatas to delete from {} catalog of {} workgroup".
format(source_cat_uuid, wg_uuid))
to_delete = isogeo.search(group=wg_uuid,
whole_results=True,
query="catalog:{}".format(source_cat_uuid))
# listing
li_dlt_uuid = [
md.get("_id") for md in to_delete.results
if migrated_cat_tag not in md.get("tags")
]
# ################# BACKUP MDs THAT ARE GONNA BE DELETED #######################
# instanciate backup manager
if environ.get("BACKUP") == "1" and len(li_dlt_uuid) > 0:
logger.info(
"====== {} source metadatas listed gonna be backuped then deleted ======"
.format(len(li_dlt_uuid)))
logger.info(
"-------------------------- BACKUP -------------------------------"
from isogeotoxlsx import Isogeo2xlsx
load_dotenv("dev.env", override=True)
isogeo = Isogeo(
auth_mode="group",
client_id=environ.get("ISOGEO_API_GROUP_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_GROUP_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
# getting a token
isogeo.connect()
search = isogeo.search(include="all", share="1258fd9c21e347a6b6235b911826a798")
isogeo.close()
# print(search)
# instanciate the final workbook
out_workbook = Isogeo2xlsx(lang=isogeo.lang,
url_base_edit=isogeo.app_url,
url_base_view=isogeo.oc_url)
# add needed worksheets
out_workbook.set_worksheets(auto=search.tags.keys())
# parse search results
for md in map(Metadata.clean_attributes, search.results):
out_workbook.store_metadatas(md)
pass
# ----------------------------------- MIGRATING ------------------------------------
logger.info(
"--------------------------- MIGRATING -------------------------------"
)
# Preparing to exclude "M_Source" catalog
li_cat_to_exclude = []
if environ.get("ISOGEO_CATALOG_SOURCE"):
li_cat_to_exclude.append(environ.get("ISOGEO_CATALOG_SOURCE"))
# Preparing to check if a target metadata has already been migrated
if environ.get("ISOGEO_CATALOG_MIGRATED"):
already_migrated_search = isogeo.search(
group=environ.get("ISOGEO_ORIGIN_WORKGROUP"),
whole_results=True,
query="catalog:{}".format(environ.get("ISOGEO_CATALOG_MIGRATED")))
li_already_migrated_uuid = [
md.get("_id") for md in already_migrated_search.results
]
else:
logger.error(
"'ISOGEO_CATALOG_MIGRATED' environment variable has to be set before proceeding further"
)
exit()
# Let's start migration
li_migrated = []
li_failed = []
index = 0
for to_migrate in li_to_migrate:
share_token = config.get('auth', 'app_secret')
# ------------ Real start ----------------
# instanciating the class
isogeo = Isogeo(client_id=share_id,
client_secret=share_token)
# check which sub resources are available
print(isogeo.sub_resources_available)
# getting a token
jeton = isogeo.connect()
# let's search for metadatas!
print(dir(isogeo))
search = isogeo.search(jeton)
print(sorted(search.keys()))
print(search.get('query'))
print("Total count of metadatas shared: ", search.get("total"))
print("Count of resources got by request: {}\n".format(len(search.get("results"))))
# get one random resource
hatnumber = randrange(0, len(search.get("results")))
my_resource = isogeo.resource(jeton,
search.get("results")[hatnumber].get("_id"),
sub_resources=isogeo.sub_resources_available,
prot="https"
)
print(sorted(my_resource.keys()))
isogeo = Isogeo(
auth_mode="group",
client_id=environ.get("ISOGEO_API_GROUP_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_GROUP_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
# getting a token
isogeo.connect()
# ------------ Isogeo search --------------------------
search_results = isogeo.search(
include="all",
specific_md=(
"70f1192f67ac43e5987800ead18effb2",
"b140d9a92c20416d97c3cdc12dc12607",
),
)
isogeo.close() # close session
# ------------ REAL START ----------------------------
# output folder
Path("_output/").mkdir(exist_ok=True)
# template
template_path = Path("tests/fixtures/template_Isogeo.docx")
assert template_path.is_file()
# thumbnails table
thumbnails_table_csv_path = Path("tests/fixtures/thumbnails.csv")
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
logger.info(
"Retrieving metadatas to delete from {} catalog of {} workgroup".
format(cat_deletion_target, wg_deletion_target))
to_delete = isogeo.search(group=wg_deletion_target,
whole_results=True,
query="catalog:{}".format(cat_deletion_target))
# listing
li_dlt_uuid = [md.get("_id") for md in to_delete.results]
# ################# BACKUP MDs THAT ARE GONNA BE DELETED #######################
# instanciate backup manager
if environ.get("BACKUP") == "1" and len(li_dlt_uuid) > 0:
logger.info(
"====== {} source metadatas listed gonna be backuped then deleted ======"
.format(len(li_dlt_uuid)))
logger.info(
"-------------------------- BACKUP -------------------------------"
)
# backup manager instanciation
# print(dico_gs)
# print(dico_gs.keys())
# print(dico_gs.get('ayants-droits')[1].get("BD_TOPO_2015_VOIES_FERREES_ET_AUTRES"))
# print(dico_gs.get('ayants-droits')[1].get("bd_topo_reseau_routier_route_primaire"))
# print(dico_gs.get('ayants-droits')[1].keys())
# print(dico_gs.get('layers'))
# ------------------------------------------------------------------------
# ------------ ISOGEO ----------------------------------------------------
# instanciating the class
isogeo = Isogeo(client_id=app_id, client_secret=app_secret, lang=app_lang)
token = isogeo.connect()
search_results = isogeo.search(token)
search_results = search_results.get('results')
# ------------------------------------------------------------------------
# ## EXCELs ############
# -- WMS -------------------------------------------------------
wb_gs_full = Workbook()
dest_gs_full = '{}_gs_full.xlsx'.format(out_prefix)
ws_gs_full = wb_gs_full.active
ws_gs_full.title = "GEOSERVER - FULL"
ws_gs_full["A1"] = "GS_WORKSPACE"
ws_gs_full["B1"] = "GS_DATASTORE_NAME"
ws_gs_full["C1"] = "GS_DATASTORE_TYPE"
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
# Ask Isogeo API about metadatas from relevant catalog
bd_carto_search = isogeo.search(
group=ignf_wg_uuid,
query="catalog:{}".format(bdcarto32_cat_uuid),
whole_results=True
)
logger.info("{} metadatas retrieved from {} catalog".format(bd_carto_search.total, bdcarto32_cat_uuid))
# Browsing metadatas retrieved from Isogeo API and updating their title
md_parsed_count = 0
for md in bd_carto_search.results:
# refresh token if needed
if default_timer() - auth_timer >= 6900:
logger.info("Manually refreshing token")
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
logger.info("Retrieving metadatas to delete from {} workgroup".format(wg_deletion_target))
to_delete = isogeo.search(
group=wg_deletion_target,
whole_results=True
)
to_keep_tag = "catalog:{}".format(cat_to_keep)
# listing
li_dlt_uuid = [md.get("_id") for md in to_delete.results if to_keep_tag not in md.get("tags")]
# ################# BACKUP MDs THAT ARE GONNA BE DELETED #######################
# instanciate backup manager
if environ.get("BACKUP") == "1" and len(li_dlt_uuid) > 0:
logger.info("========= {} source metadatas listed gonna be backuped then deleted =========".format(len(li_dlt_uuid)))
logger.info("-------------------------- BACKUP -------------------------------")
# backup manager instanciation
backup_path = Path(r"./scripts/misc/delete_md/_output/_backup")
backup_mng = BackupManager(api_client=isogeo, output_folder=backup_path)
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
logger.info(
"------------------------- MATCHING SESSION --------------------------"
)
# retrieve destination worgroup metadata
destination_search = isogeo.search(group=origin_wg_uuid,
whole_results=True)
li_dest_md = destination_search.results
origin_wg_name = li_dest_md[0].get("_creator").get("contact").get("name")
logger.info(
"{} metadatas retrieved from '{}' destination workgroup ({})".format(
destination_search.total, origin_wg_name, origin_wg_uuid))
# retrieve isogeo catalogs infos
li_ignf_wg_cat = isogeo.catalog.listing(workgroup_id=ignf_wg_uuid,
include="all")
li_ignf_cat_uuid = list(target_matching_dict.keys())
li_ignf_cat = []
for cat in li_ignf_wg_cat:
if cat.get("_id") in li_ignf_cat_uuid:
li_ignf_cat.append(Catalog.clean_attributes(cat))
else:
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_spinner.text = "Authentication to Isogeo API succeed"
auth_spinner.succeed()
# ###################################### ISOGEO API SEARCH #####################################
search_spinner = Halo(
text="Searching for '{}' workgroup metadatas".format(wg_uuid),
spinner='dots')
search_spinner.start()
# Search about workgroup service metadatas
md_search = isogeo.search(group=wg_uuid, include="all", whole_results=True)
isogeo.close()
# ##################################### PARSE SEARCH RESULT ####################################
li_service_md = []
li_data_md = []
for md in md_search.results:
metadata = Metadata.clean_attributes(md)
if md.get("type") == "service":
li_service_md.append(metadata)
else:
li_data_md.append(metadata)
succeed_task_msg = "{} metadatas retrieved, including {} service metadatas and {} other type metadatas".format(
md_search.total, len(li_service_md), len(li_data_md))
# API client instanciation
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
# load source metadatas' uuid, title and name
src_md_search = isogeo.search(
group=ign_wg_uuid,
query="catalog:{}".format(source_cat_uuid),
whole_results=True
)
li_md_src = []
li_name_src = []
for md in src_md_search.results:
li_md_src.append((md.get("_id"), md.get("title"), md.get("name")))
li_name_src.append(md.get("name"))
# load target metadatas' uuid, title and name
trg_md_search = isogeo.search(
group=amp_wg_uuid,
query="catalog:{}".format(target_cat_uuid),
whole_results=True
)
li_md_trg = []
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
src_md = isogeo.search(
group=environ.get("ISOGEO_ORIGIN_WORKGROUP"),
whole_results=True,
query="catalog:{}".format(src_cat),
include="all"
)
# listing
li_md_to_delete = []
for md in src_md.results:
metadata = Metadata.clean_attributes(md)
md_cat = [metadata.tags.get(tag) for tag in metadata.tags if tag.startswith("catalog:")]
if trg_cat not in md_cat:
li_md_to_delete.append(metadata._id)
else:
pass
logger.info("------- {} source metadatas listed gonna be backuped then deleted -------".format(len(li_md_to_delete)))
# ################# BACKUP MDs THAT ARE GONNA BE DELETED #######################
# instanciate backup manager
print("Working with package version: {}".format(__version__))
utils = IsogeoUtils()
#print(dir(utils))
#print(help(utils))
#######
api_auth = utils.credentials_loader(r"./client_secrets.json")
# authenticate your client application
isogeo = Isogeo(client_id=api_auth.get("client_id"),
client_secret=api_auth.get("client_secret"))
# get the token
token = isogeo.connect()
# add properties as attribute
isogeo.get_app_properties(token)
# set augment option on True
search = isogeo.search(token,
query="bar",
page_size=0,
whole_share=0,
augment=1)
print(search.get("total"))
print(search.get("tags"))
print(isogeo.shares_id)
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
workgroup_uuid = environ.get("ISOGEO_ORIGIN_WORKGROUP")
trg_cat_uuid = environ.get("ISOGEO_CATALOG_TARGET")
# Search about all workgroup metadatas because there are less than 800
whole_md_search = isogeo.search(group=workgroup_uuid, whole_results=True)
isogeo.close()
print("{} metadatas loaded from {} workgroup".format(
whole_md_search.total, workgroup_uuid))
trg_cat_tag = "catalog:{}".format(trg_cat_uuid)
li_src_md = whole_md_search.results
li_trg_md = [
md for md in whole_md_search.results
if trg_cat_tag in list(md.get("tags").keys())
]
for md in li_trg_md:
li_src_md.remove(md)
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
# ################# RETRIEVE UUID FROM ISOGEO API REQUEST #################
ign_wg_uuid = environ.get("ISOGEO_IGN_WORKGROUP")
raster_cat_uuid = environ.get("IGN_RASTER_CATALOG_UUID")
raster_cat_search = isogeo.search(
group=ign_wg_uuid,
query="catalog:{}".format(raster_cat_uuid),
whole_results=True)
logger.info(
"{} metadata retrieved to import into target work group".format(
raster_cat_search.total))
# to store source metadata uuid
li_uuid_to_migrate = []
# prepare csv reading
for md in raster_cat_search.results:
src_uuid = md.get("_id")
# check source UUID validity
if not checker.check_is_uuid(src_uuid):
logger.info("{} source UUID isn't valid".format(src_uuid))
else:
li_uuid_to_migrate.append(src_uuid)
# API client instanciation
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
# ask Isogeo API about whole MAMP metadatas
whole_search = isogeo.search(whole_results=True, group=wg_uuid)
# retrieve the dedicated catalog object
cat = isogeo.catalog.get(workgroup_id=wg_uuid,
catalog_id="d220c42c6b4c4dbd8b068dde32579b58")
logger.info("{} metadata retrieved from {} workgroup".format(
whole_search.total, wg_uuid))
# filter involved metadatas
li_md = [md for md in whole_search.results if md.get("name")]
li_md_to_parse_infos = [(md.get("_id"), md.get("name")) for md in li_md]
li_for_csv = []
nb_matchs = 0
for tup in li_md_tup:
uuid = tup[0]
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
lic_etalab1_uuid = "63f121e14eda4f47b748595e0bcccc31"
lic_etalab2_uuid = "f6e0c665905a4feab1e9c1d6359a225f"
lic_etalab1 = isogeo.license.get(lic_etalab1_uuid)
lic_etalab2 = isogeo.license.get(lic_etalab2_uuid)
new_condition = Condition()
new_condition._license = lic_etalab2
search_migrated = isogeo.search(
group=origin_wg_uuid,
query="catalog:{}".format(migrated_cat_uuid),
whole_results=True,
include=("conditions",)
)
for md in search_migrated.results:
# retrieve licenses'ids of metadata's conditions
md_lic_uuid = [condition.get("license").get("_id") for condition in md.get("conditions")]
# tests if Etalab is one of metadata conditions' license
if len(md_lic_uuid) and lic_etalab1_uuid in md_lic_uuid:
# retrieve condition to delete dict
md_condition = [condition for condition in md.get("conditions") if condition.get("license").get("_id") == lic_etalab1_uuid][0]
# build metadata object
isogeo_md = Metadata(**md)
# add condtion to delete descritpion to new condition before adding new condition
logger.info("Authenticating to Isogeo API")
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
auth_mode="user_legacy",
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
auth_timer = default_timer()
trg_migrated_search = isogeo.search(
group=wg_uuid,
whole_results=True,
query="catalog:{}".format(migrated_cat_uuid))
li_trg_migrated_uuid = [
md.get("_id") for md in trg_migrated_search.results
]
# Build the list of metadata to delete UUID
li_dlt_uuid = []
# but first, check if each source md has actually been migrated
for migrated in li_migrated:
if migrated[1] in li_trg_migrated_uuid:
li_dlt_uuid.append(migrated[0])
else:
pass
# ################# BACKUP MDs THAT ARE GONNA BE DELETED #######################
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
workgroup_uuid = environ.get("ISOGEO_ORIGIN_WORKGROUP")
# Search about workgroup service metadatas
service_md_search = isogeo.search(group=workgroup_uuid,
query="type:service",
include="all",
whole_results=True)
isogeo.close()
print("{} service metadata retrieved from {} workgroup".format(
service_md_search.total, workgroup_uuid))
li_for_csv = []
for md in service_md_search.results:
service_md_uuid = md.get("_id")
service_md_title = md.get("title")
associated_layer_counter = 0
if len(md.get("layers")):
for layer in md.get("layers"):
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
workgroup_uuid = environ.get("ISOGEO_ORIGIN_WORKGROUP")
# Search about all workgroup metadatas because there are less than 800
whole_md_search = isogeo.search(group=workgroup_uuid,
whole_results=True,
include="all")
isogeo.close()
li_md = [md for md in whole_md_search.results if md.get("name")]
print("{} metadatas loaded from {} workgroup".format(
whole_md_search.total, workgroup_uuid))
li_md_alias = []
for md in li_md:
if md.get("name"):
md_alias = md.get("name") + " --> " + md.get("path")
if md_alias not in li_md_alias:
li_md_alias.append(md_alias)
else:
# instanciating the class
isogeo = Isogeo(client_id=share_id,
client_secret=share_token)
token = isogeo.connect()
# ------------ REAL START ----------------------------
# opening a geojson file
gson_input = r'samples_boundingbox.geojson'
with open(gson_input) as data_file:
data = json.load(data_file)
# search & compare
basic_search = isogeo.search(token,
page_size=0,
whole_share=0)
print("Comparing count of results returned: ")
print("\t- without any filter = ", basic_search.get('total'))
for feature in data['features']:
# just for VIPolygons
if feature['geometry']['type'] != "Polygon":
print("Geometry type must be a polygon")
continue
else:
pass
# search & display results
bbox = str(feature['bbox'])[1:-1]
filtered_search = isogeo.search(token,