def export_archive(datas, archive_path, json_settings):
# Not Finished
export_type = json_global_settings["export_type"]
if export_type == "json":
archive_path = os.path.join(archive_path + ".json")
if os.path.exists(archive_path):
datas2 = ofrenamer.start(archive_path, json_settings)
if datas == datas2:
return
with open(archive_path, 'w') as outfile:
json.dump(datas, outfile)
if export_type == "csv":
archive_path = os.path.join(archive_path + ".csv")
with open(archive_path, mode='w', encoding='utf-8',
newline='') as csv_file:
for data in datas:
fieldnames = []
media_type = data["type"].lower()
valid = list(chain.from_iterable(data["valid"]))
invalid = list(chain.from_iterable(data["invalid"]))
if valid:
fieldnames.extend(valid[0].keys())
elif invalid:
fieldnames.extend(invalid[0].keys())
header = [media_type] + fieldnames
if len(fieldnames) > 1:
writer = csv.DictWriter(csv_file, fieldnames=header)
writer.writeheader()
for item in valid:
writer.writerow({**{media_type: "valid"}, **item})
for item in invalid:
writer.writerow({**{media_type: "invalid"}, **item})
def process_metadata(api: start, new_metadata, formatted_directories, subscription, api_type, api_path, archive_path, site_name):
print("Processing metadata.")
legacy_metadata_object = legacy_metadata_fixer(
formatted_directories, api)
new_metadata_object = create_metadata(
api, new_metadata, standard_format=True)
if legacy_metadata_object:
print("Merging new metadata with legacy metadata.")
new_metadata_object = compare_metadata(
new_metadata_object, legacy_metadata_object)
old_metadata_set = import_archive(archive_path)
old_metadata_object = create_metadata(
api, old_metadata_set, api_type=api_type)
if old_metadata_object:
print("Merging new metadata with old metadata.")
new_metadata_object = compare_metadata(
new_metadata_object, old_metadata_object)
if not subscription.download_info:
subscription.download_info["metadata_locations"] = {}
subscription.download_info["directory"] = j_directory
subscription.download_info["webhook"] = webhook
subscription.download_info["metadata_locations"][api_type] = archive_path
subscription.set_scraped(api_type, new_metadata_object)
print("Renaming files.")
new_metadata_object = ofrenamer.start(
subscription, api_type, api_path, site_name, json_settings)
subscription.set_scraped(api_type, new_metadata_object)
print("Finished processing metadata.")
return new_metadata_object
def process_metadata(archive_path, new_metadata_object, site_name, parent_type, api_path, subscription, delete_metadatas):
Session, api_type, folder = main_helper.export_sqlite(
archive_path, new_metadata_object, parent_type)
if not subscription.download_info:
subscription.download_info["metadata_locations"] = {}
subscription.download_info["directory"] = j_directory
subscription.download_info["webhook"] = webhook
database_name = parent_type if parent_type else api_type
subscription.download_info["metadata_locations"][api_type] = {}
subscription.download_info["metadata_locations"][api_type][database_name] = archive_path
print("Renaming files.")
new_metadata_object = ofrenamer.start(
Session, parent_type, api_type, api_path, site_name, subscription, folder, json_settings)
for old_metadata in delete_metadatas:
if os.path.exists(old_metadata):
os.remove(old_metadata)
def export_archive(datas,
archive_directory,
json_settings,
rename=True,
legacy_directory=""):
if os.path.exists(legacy_directory):
shutil.rmtree(legacy_directory)
if json_settings["export_metadata"]:
export_type = json_global_settings["export_type"]
if export_type == "json":
os.makedirs(os.path.dirname(archive_directory), exist_ok=True)
archive_path = archive_directory + ".json"
if ".json" in archive_directory:
archive_path = archive_directory
if os.path.exists(archive_path) and rename:
datas2 = ofrenamer.start(archive_path, json_settings)
if datas == datas2:
return
with open(archive_path, 'w', encoding='utf-8') as outfile:
json.dump(datas, outfile, indent=2)
def process_metadata(api: start, new_metadata, formatted_directories,
subscription, api_type, api_path, archive_path,
site_name):
legacy_metadata_object = legacy_metadata_fixer(formatted_directories, api)
new_metadata_object = prepare_metadata(new_metadata, api=api).metadata
new_metadata_object = compare_metadata(new_metadata_object,
legacy_metadata_object)
old_metadata_set = import_archive(archive_path)
old_metadata_object = prepare_metadata(old_metadata_set, api=api).metadata
new_metadata_object = compare_metadata(new_metadata_object,
old_metadata_object)
if not subscription.download_info:
subscription.download_info["metadata_locations"] = {}
subscription.download_info["directory"] = j_directory
subscription.download_info["webhook"] = webhook
subscription.download_info["metadata_locations"][api_type] = archive_path
subscription.set_scraped(api_type, new_metadata_object)
new_metadata_object = ofrenamer.start(subscription, api_type, api_path,
site_name, json_settings)
subscription.set_scraped(api_type, new_metadata_object)
return new_metadata_object
