def process_metadata(api: start, new_metadata, formatted_directories, subscription, api_type, api_path, archive_path, site_name): print("Processing metadata.") legacy_metadata_object = legacy_metadata_fixer( formatted_directories, api) new_metadata_object = create_metadata( api, new_metadata, standard_format=True) if legacy_metadata_object: print("Merging new metadata with legacy metadata.") new_metadata_object = compare_metadata( new_metadata_object, legacy_metadata_object) old_metadata_set = import_archive(archive_path) old_metadata_object = create_metadata( api, old_metadata_set, api_type=api_type) if old_metadata_object: print("Merging new metadata with old metadata.") new_metadata_object = compare_metadata( new_metadata_object, old_metadata_object) if not subscription.download_info: subscription.download_info["metadata_locations"] = {} subscription.download_info["directory"] = j_directory subscription.download_info["webhook"] = webhook subscription.download_info["metadata_locations"][api_type] = archive_path subscription.set_scraped(api_type, new_metadata_object) print("Renaming files.") new_metadata_object = ofrenamer.start( subscription, api_type, api_path, site_name, json_settings) subscription.set_scraped(api_type, new_metadata_object) print("Finished processing metadata.") return new_metadata_object
def legacy_metadata_fixer(formatted_directories: dict, api: object) -> tuple[create_metadata, list]: delete_legacy_metadatas = [] legacy_metadatas = formatted_directories["legacy_metadatas"] new_metadata_directory = formatted_directories["metadata_directory"] old_metadata_directory = os.path.dirname( legacy_metadatas["legacy_metadata"]) metadata_name = os.path.basename(f"{old_metadata_directory}.json") q = [] for key, legacy_directory in legacy_metadatas.items(): if legacy_directory == formatted_directories["metadata_directory"]: continue if os.path.exists(legacy_directory): folders = os.listdir(legacy_directory) metadata_names = media_types() metadata_names = [f"{k}.json" for k, v in metadata_names] print type_one_files = main_helper.remove_mandatory_files( folders, keep=metadata_names) new_format = [] for type_one_file in type_one_files: api_type = type_one_file.removesuffix(".json") legacy_metadata_path = os.path.join( legacy_directory, type_one_file) legacy_metadata = import_archive(legacy_metadata_path) if legacy_metadata: delete_legacy_metadatas.append(legacy_metadata_path) legacy_metadata = create_metadata( api, legacy_metadata, api_type=api_type).convert() new_format.append(legacy_metadata) new_format = dict( merge({}, *new_format, strategy=Strategy.ADDITIVE)) old_metadata_object = create_metadata(api, new_format) if legacy_directory != new_metadata_directory: import_path = os.path.join(legacy_directory, metadata_name) new_metadata_set = import_archive( import_path) if new_metadata_set: new_metadata_object2 = create_metadata( api, new_metadata_set) old_metadata_object = compare_metadata( new_metadata_object2, old_metadata_object) q.append(old_metadata_object) print print results = create_metadata() for merge_into in q: print results = compare_metadata( results, merge_into) print print return results, delete_legacy_metadatas
def process_legacy_metadata(api: start, new_metadata_set, formatted_directories, subscription, api_type, api_path, archive_path, site_name): print("Processing metadata.") delete_metadatas = [] archive_path = archive_path.replace("db", "json") legacy_metadata_object, delete_legacy_metadatas = legacy_metadata_fixer( formatted_directories, api) if delete_legacy_metadatas: print("Merging new metadata with legacy metadata.") old_metadata_set = import_archive(archive_path) old_metadata_object = create_metadata( api, old_metadata_set, api_type=api_type) if old_metadata_set: print("Merging new metadata with old metadata.") old_metadata_object = compare_metadata( old_metadata_object, legacy_metadata_object) old_metadata_set = [] for media_type, value in old_metadata_object.content: for status, value2 in value: for value3 in value2: x = value3.medias item = value3.convert(keep_empty_items=True) old_metadata_set.append(item) print print print subscription.set_scraped(api_type, old_metadata_object) if old_metadata_set: delete_metadatas.append(archive_path) final_set = [] for item in old_metadata_set: x = [x for x in new_metadata_set if x["post_id"] == item["post_id"]] if not x: final_set.append(item) print print print("Finished processing metadata.") return final_set, delete_metadatas