for row in reader: row_num += 1 src_uuid = row.get("source_uuid") src_title = row.get("source_title") src_name = row.get("source_name") trg_name = row.get("target_name") trg_uuid = row.get("target_uuid") if src_uuid != "source_uuid" and src_uuid != "61d7c106e5eb4dc086a4a5c3b08ad4aa": # PROD # if src_uuid == "61d7c106e5eb4dc086a4a5c3b08ad4aa": # DEV src_found.append(src_uuid) trg_found.append(trg_uuid) # check if the target metadata exists if trg_uuid == "NR": logger.info("l.{} - there is no target".format(row_num)) # check source UUID validity elif not checker.check_is_uuid(src_uuid): logger.info("l.{} - {} source UUID isn't valid".format( row_num, src_uuid)) # check if source UUID appears just one time in the field elif li_src_to_migrate.count(src_uuid) > 0: logger.info( "l.{} - {} already exist in the tab at line {}".format( row_num, src_uuid, str(src_found.index(src_uuid) + 1))) # if UUID, title and name of source metadata have passed all checks, # time to test UUID and nam of target metadata else: # check target UUID validity if not checker.check_is_uuid(trg_uuid): logger.info("l.{} -{} target UUID isn't valid".format( row_num, trg_uuid))
# read csv file # structure CSV : title;old_name;old_uri;new_name;new_uri input_csv = Path(r"./input/sample_migration_import.csv") # parse input CSV and perform operations logger.info("Start reading the input file") with input_csv.open() as csvfile: # read it fieldnames = ["title", "old_name", "old_uri", "new_name", "new_uri"] reader = csv.DictReader(csvfile, delimiter=";", fieldnames=fieldnames) # parse csv for row in reader: # check uuids if not checker.check_is_uuid(row.get("old_uri")): logger.error( "UUID '{}' is not correct. It'll be ignored.".format( row.get("old_uri"))) continue if not checker.check_is_uuid(row.get("old_uri")): logger.error( "UUID '{}' is not correct. It'll be ignored.".format( row.get("old_uri"))) continue # prepare backup li_to_backup.extend([row.get("old_uri"), row.get("new_uri")]) # prepare migration li_to_migrate.append((row.get("old_uri"), row.get("new_uri")))
raster_cat_uuid = environ.get("IGN_RASTER_CATALOG_UUID") raster_cat_search = isogeo.search( group=ign_wg_uuid, query="catalog:{}".format(raster_cat_uuid), whole_results=True) logger.info( "{} metadata retrieved to import into target work group".format( raster_cat_search.total)) # to store source metadata uuid li_uuid_to_migrate = [] # prepare csv reading for md in raster_cat_search.results: src_uuid = md.get("_id") # check source UUID validity if not checker.check_is_uuid(src_uuid): logger.info("{} source UUID isn't valid".format(src_uuid)) else: li_uuid_to_migrate.append(src_uuid) # li_uuid_to_migrate = ["96cd73589f474565b35ce59b12633c3b"] # ############################################################# # ############################### MIGRATING ############################### logger.info("{} metadatas will be migrated".format( len(li_uuid_to_migrate))) # ------------------------------------ BACKUP -------------------------------------- if int(environ.get("BACKUP")) == 1 and len(li_uuid_to_migrate) > 0: logger.info( "---------------------------- BACKUP {} metadatas---------------------------------" .format(len(li_uuid_to_migrate))) # backup manager instanciation backup_path = Path(r"./scripts/AMP/md_ref/rasters/_output/_backup")
with input_csv.open() as csvfile: reader = csv.DictReader(csvfile, delimiter=";", fieldnames=fieldnames) row_num = 0 for row in reader: row_num += 1 data_name = row.get("data_name") md_uuid_1 = row.get("md_uuid_1") md_uuid_2 = row.get("md_uuid_2") match_type = row.get("match_type") if data_name != "data_name": # check if the target metadata exists if md_uuid_2 == "no_match": logger.info("l.{} - there is no target".format(row_num)) # check source UUID validity elif not checker.check_is_uuid(md_uuid_1): logger.info("l.{} - {} source UUID isn't valid".format( row_num, md_uuid_1)) # check if source UUID appears just one time in the field elif li_uuid_1.count(md_uuid_1) > 0: logger.info( "l.{} - {} already exist in the tab at line {}".format( row_num, md_uuid_1, str(li_uuid_1.index(md_uuid_1) + 1))) # if UUID, title and name of source metadata have passed all checks, # time to test UUID and nam of target metadata else: # check target UUID validity if not checker.check_is_uuid(md_uuid_2): logger.info("l.{} - {} source UUID isn't valid".format( row_num, md_uuid_2))
# for row in ws_vectors.rows: # for cell in row: # print(row[39].value) # metadata UUID # print(row[0].value) # title # print(row[2].value) # abstract # print(row[5].value) # keywords # print(row[6].value) # INSPIRE # pass for row in ws_vectors.iter_rows(min_row=2): logger.info("Reading metadata row: " + row[dct_i2o_struct.get("uuid")].value) # get metadata UUID from workbook metadata_uuid = row[dct_i2o_struct.get("uuid")].value if row[dct_i2o_struct.get("uuid")] is not None and checker.check_is_uuid( metadata_uuid): metadata_uuid = row[dct_i2o_struct.get("uuid")].value else: logger.error("Invalid metadata UUID spotted: " + metadata_uuid) continue # compare title and technical names if row[dct_i2o_struct.get("title")].value == row[dct_i2o_struct.get( "name")].value: logger.warning( "Row has been ignored because title has not been changed.") continue else: pass # retrieve the metadata from Isogeo
isogeo = Isogeo( client_id=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_ID"), client_secret=environ.get("ISOGEO_API_USER_LEGACY_CLIENT_SECRET"), auth_mode="user_legacy", auto_refresh_url="{}/oauth/token".format(environ.get("ISOGEO_ID_URL")), platform=environ.get("ISOGEO_PLATFORM", "qa"), ) isogeo.connect( username=environ.get("ISOGEO_USER_NAME"), password=environ.get("ISOGEO_USER_PASSWORD"), ) src_cat_uuid = environ.get("ISOGEO_CATALOG_SOURCE") trg_cat_uuid = environ.get("ISOGEO_CATALOG_TARGET") # SOURCES if checker.check_is_uuid(src_cat_uuid): src_md = isogeo.search(group=environ.get("ISOGEO_ORIGIN_WORKGROUP"), query="catalog:{}".format(src_cat_uuid), whole_results=True) print("{} source metadata loaded".format(src_md.total)) content = src_md.results with open("scripts/dijon/migration/output_src.json", "w") as outfile: json.dump(content, outfile, sort_keys=True, indent=4) else: print("wrong source catalog UUID : {}".format(src_cat_uuid)) # TARGET if checker.check_is_uuid(trg_cat_uuid): trg_md = isogeo.search(group=environ.get("ISOGEO_ORIGIN_WORKGROUP"), query="catalog:{}".format(trg_cat_uuid),