for row in reader:
row_num += 1
src_uuid = row.get("source_uuid")
src_title = row.get("source_title")
src_name = row.get("source_name")
trg_name = row.get("target_name")
trg_uuid = row.get("target_uuid")
if src_uuid != "source_uuid" and src_uuid != "61d7c106e5eb4dc086a4a5c3b08ad4aa": # PROD
# if src_uuid == "61d7c106e5eb4dc086a4a5c3b08ad4aa": # DEV
src_found.append(src_uuid)
trg_found.append(trg_uuid)
# check if the target metadata exists
if trg_uuid == "NR":
logger.info("l.{} - there is no target".format(row_num))
# check source UUID validity
elif not checker.check_is_uuid(src_uuid):
logger.info("l.{} - {} source UUID isn't valid".format(
row_num, src_uuid))
# check if source UUID appears just one time in the field
elif li_src_to_migrate.count(src_uuid) > 0:
logger.info(
"l.{} - {} already exist in the tab at line {}".format(
row_num, src_uuid,
str(src_found.index(src_uuid) + 1)))
# if UUID, title and name of source metadata have passed all checks,
# time to test UUID and nam of target metadata
else:
# check target UUID validity
if not checker.check_is_uuid(trg_uuid):
logger.info("l.{} -{} target UUID isn't valid".format(
row_num, trg_uuid))
# read csv file
# structure CSV : title;old_name;old_uri;new_name;new_uri
input_csv = Path(r"./input/sample_migration_import.csv")
# parse input CSV and perform operations
logger.info("Start reading the input file")
with input_csv.open() as csvfile:
# read it
fieldnames = ["title", "old_name", "old_uri", "new_name", "new_uri"]
reader = csv.DictReader(csvfile, delimiter=";", fieldnames=fieldnames)
# parse csv
for row in reader:
# check uuids
if not checker.check_is_uuid(row.get("old_uri")):
logger.error(
"UUID '{}' is not correct. It'll be ignored.".format(
row.get("old_uri")))
continue
if not checker.check_is_uuid(row.get("old_uri")):
logger.error(
"UUID '{}' is not correct. It'll be ignored.".format(
row.get("old_uri")))
continue
# prepare backup
li_to_backup.extend([row.get("old_uri"), row.get("new_uri")])
# prepare migration
li_to_migrate.append((row.get("old_uri"), row.get("new_uri")))
raster_cat_uuid = environ.get("IGN_RASTER_CATALOG_UUID")
raster_cat_search = isogeo.search(
group=ign_wg_uuid,
query="catalog:{}".format(raster_cat_uuid),
whole_results=True)
logger.info(
"{} metadata retrieved to import into target work group".format(
raster_cat_search.total))
# to store source metadata uuid
li_uuid_to_migrate = []
# prepare csv reading
for md in raster_cat_search.results:
src_uuid = md.get("_id")
# check source UUID validity
if not checker.check_is_uuid(src_uuid):
logger.info("{} source UUID isn't valid".format(src_uuid))
else:
li_uuid_to_migrate.append(src_uuid)
# li_uuid_to_migrate = ["96cd73589f474565b35ce59b12633c3b"] # #############################################################
# ############################### MIGRATING ###############################
logger.info("{} metadatas will be migrated".format(
len(li_uuid_to_migrate)))
# ------------------------------------ BACKUP --------------------------------------
if int(environ.get("BACKUP")) == 1 and len(li_uuid_to_migrate) > 0:
logger.info(
"---------------------------- BACKUP {} metadatas---------------------------------"
.format(len(li_uuid_to_migrate)))
# backup manager instanciation
backup_path = Path(r"./scripts/AMP/md_ref/rasters/_output/_backup")
with input_csv.open() as csvfile:
reader = csv.DictReader(csvfile, delimiter=";", fieldnames=fieldnames)
row_num = 0
for row in reader:
row_num += 1
data_name = row.get("data_name")
md_uuid_1 = row.get("md_uuid_1")
md_uuid_2 = row.get("md_uuid_2")
match_type = row.get("match_type")
if data_name != "data_name":
# check if the target metadata exists
if md_uuid_2 == "no_match":
logger.info("l.{} - there is no target".format(row_num))
# check source UUID validity
elif not checker.check_is_uuid(md_uuid_1):
logger.info("l.{} - {} source UUID isn't valid".format(
row_num, md_uuid_1))
# check if source UUID appears just one time in the field
elif li_uuid_1.count(md_uuid_1) > 0:
logger.info(
"l.{} - {} already exist in the tab at line {}".format(
row_num, md_uuid_1,
str(li_uuid_1.index(md_uuid_1) + 1)))
# if UUID, title and name of source metadata have passed all checks,
# time to test UUID and nam of target metadata
else:
# check target UUID validity
if not checker.check_is_uuid(md_uuid_2):
logger.info("l.{} - {} source UUID isn't valid".format(
row_num, md_uuid_2))
# for row in ws_vectors.rows:
# for cell in row:
# print(row[39].value) # metadata UUID
# print(row[0].value) # title
# print(row[2].value) # abstract
# print(row[5].value) # keywords
# print(row[6].value) # INSPIRE
# pass
for row in ws_vectors.iter_rows(min_row=2):
logger.info("Reading metadata row: " +
row[dct_i2o_struct.get("uuid")].value)
# get metadata UUID from workbook
metadata_uuid = row[dct_i2o_struct.get("uuid")].value
if row[dct_i2o_struct.get("uuid")] is not None and checker.check_is_uuid(
metadata_uuid):
metadata_uuid = row[dct_i2o_struct.get("uuid")].value
else:
logger.error("Invalid metadata UUID spotted: " + metadata_uuid)
continue
# compare title and technical names
if row[dct_i2o_struct.get("title")].value == row[dct_i2o_struct.get(
"name")].value:
logger.warning(
"Row has been ignored because title has not been changed.")
continue
else:
pass
# retrieve the metadata from Isogeo
isogeo = Isogeo(
client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"),
client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"),
auth_mode="user_legacy",
auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")),
platform=environ.get("ISOGEO_PLATFORM", "qa"),
)
isogeo.connect(
username=environ.get("ISOGEO_USER_NAME"),
password=environ.get("ISOGEO_USER_PASSWORD"),
)
src_cat_uuid = environ.get("ISOGEO_CATALOG_SOURCE")
trg_cat_uuid = environ.get("ISOGEO_CATALOG_TARGET")
# SOURCES
if checker.check_is_uuid(src_cat_uuid):
src_md = isogeo.search(group=environ.get("ISOGEO_ORIGIN_WORKGROUP"),
query="catalog:{}".format(src_cat_uuid),
whole_results=True)
print("{} source metadata loaded".format(src_md.total))
content = src_md.results
with open("scripts/dijon/migration/output_src.json", "w") as outfile:
json.dump(content, outfile, sort_keys=True, indent=4)
else:
print("wrong source catalog UUID : {}".format(src_cat_uuid))
# TARGET
if checker.check_is_uuid(trg_cat_uuid):
trg_md = isogeo.search(group=environ.get("ISOGEO_ORIGIN_WORKGROUP"),
query="catalog:{}".format(trg_cat_uuid),