def export_archive(datas, archive_path, json_settings): # Not Finished export_type = json_global_settings["export_type"] if export_type == "json": archive_path = os.path.join(archive_path + ".json") if os.path.exists(archive_path): datas2 = ofrenamer.start(archive_path, json_settings) if datas == datas2: return with open(archive_path, 'w') as outfile: json.dump(datas, outfile) if export_type == "csv": archive_path = os.path.join(archive_path + ".csv") with open(archive_path, mode='w', encoding='utf-8', newline='') as csv_file: for data in datas: fieldnames = [] media_type = data["type"].lower() valid = list(chain.from_iterable(data["valid"])) invalid = list(chain.from_iterable(data["invalid"])) if valid: fieldnames.extend(valid[0].keys()) elif invalid: fieldnames.extend(invalid[0].keys()) header = [media_type] + fieldnames if len(fieldnames) > 1: writer = csv.DictWriter(csv_file, fieldnames=header) writer.writeheader() for item in valid: writer.writerow({**{media_type: "valid"}, **item}) for item in invalid: writer.writerow({**{media_type: "invalid"}, **item})
def process_metadata(api: start, new_metadata, formatted_directories, subscription, api_type, api_path, archive_path, site_name): print("Processing metadata.") legacy_metadata_object = legacy_metadata_fixer( formatted_directories, api) new_metadata_object = create_metadata( api, new_metadata, standard_format=True) if legacy_metadata_object: print("Merging new metadata with legacy metadata.") new_metadata_object = compare_metadata( new_metadata_object, legacy_metadata_object) old_metadata_set = import_archive(archive_path) old_metadata_object = create_metadata( api, old_metadata_set, api_type=api_type) if old_metadata_object: print("Merging new metadata with old metadata.") new_metadata_object = compare_metadata( new_metadata_object, old_metadata_object) if not subscription.download_info: subscription.download_info["metadata_locations"] = {} subscription.download_info["directory"] = j_directory subscription.download_info["webhook"] = webhook subscription.download_info["metadata_locations"][api_type] = archive_path subscription.set_scraped(api_type, new_metadata_object) print("Renaming files.") new_metadata_object = ofrenamer.start( subscription, api_type, api_path, site_name, json_settings) subscription.set_scraped(api_type, new_metadata_object) print("Finished processing metadata.") return new_metadata_object
def process_metadata(archive_path, new_metadata_object, site_name, parent_type, api_path, subscription, delete_metadatas): Session, api_type, folder = main_helper.export_sqlite( archive_path, new_metadata_object, parent_type) if not subscription.download_info: subscription.download_info["metadata_locations"] = {} subscription.download_info["directory"] = j_directory subscription.download_info["webhook"] = webhook database_name = parent_type if parent_type else api_type subscription.download_info["metadata_locations"][api_type] = {} subscription.download_info["metadata_locations"][api_type][database_name] = archive_path print("Renaming files.") new_metadata_object = ofrenamer.start( Session, parent_type, api_type, api_path, site_name, subscription, folder, json_settings) for old_metadata in delete_metadatas: if os.path.exists(old_metadata): os.remove(old_metadata)
def export_archive(datas, archive_directory, json_settings, rename=True, legacy_directory=""): if os.path.exists(legacy_directory): shutil.rmtree(legacy_directory) if json_settings["export_metadata"]: export_type = json_global_settings["export_type"] if export_type == "json": os.makedirs(os.path.dirname(archive_directory), exist_ok=True) archive_path = archive_directory + ".json" if ".json" in archive_directory: archive_path = archive_directory if os.path.exists(archive_path) and rename: datas2 = ofrenamer.start(archive_path, json_settings) if datas == datas2: return with open(archive_path, 'w', encoding='utf-8') as outfile: json.dump(datas, outfile, indent=2)
def process_metadata(api: start, new_metadata, formatted_directories, subscription, api_type, api_path, archive_path, site_name): legacy_metadata_object = legacy_metadata_fixer(formatted_directories, api) new_metadata_object = prepare_metadata(new_metadata, api=api).metadata new_metadata_object = compare_metadata(new_metadata_object, legacy_metadata_object) old_metadata_set = import_archive(archive_path) old_metadata_object = prepare_metadata(old_metadata_set, api=api).metadata new_metadata_object = compare_metadata(new_metadata_object, old_metadata_object) if not subscription.download_info: subscription.download_info["metadata_locations"] = {} subscription.download_info["directory"] = j_directory subscription.download_info["webhook"] = webhook subscription.download_info["metadata_locations"][api_type] = archive_path subscription.set_scraped(api_type, new_metadata_object) new_metadata_object = ofrenamer.start(subscription, api_type, api_path, site_name, json_settings) subscription.set_scraped(api_type, new_metadata_object) return new_metadata_object