def resolver_links(source, keys, outfile=None): if source.split(".")[-1] == "ds": # This generates report dot_paths = [".eprint_id", ".official_url"] labels = ["eprint_id", "official_url"] all_metadata = get_records(dot_paths, "official", source, keys, labels) for meta in all_metadata: new = replace_string(meta, "official_url", "http://", "https://") if new: outfile.writerow([meta["eprint_id"], meta["official_url"], new]) else: # This makes changes for eprint_id in progressbar(keys, redirect_stdout=True): meta = get_eprint(source, eprint_id) # Ignore errors where the record doesn't exist if meta != None: if meta["eprint_status"] not in ["deletion", "inbox"]: new = replace_string(meta, "official_url", "http://", "https://") if new: url = ( source + "/rest/eprint/" + str(eprint_id) + "/official_url.txt" ) headers = {"content-type": "text/plain"} print(eprint_id) response = requests.put(url, data=new, headers=headers) print(response)
def file_mapping(source_collection): """Return a dictionary that maps /tindfiles/serve urls to records.""" mapping = {} dot_paths = [".electronic_location_and_access", "._Key"] keys = dataset.keys(source_collection) metadata = get_records(dot_paths, "files", source_collection, keys) for record in metadata: # Handle history records where the key is the item and revision k = record["_Key"] if "-" in k: rec_id = k.split("-")[0] else: rec_id = k # Ignore embargoed records if "electronic_location_and_access" in record: for filev in record["electronic_location_and_access"]: url = filev["uniform_resource_identifier"] # name = filev['electronic_name'][0] if url not in mapping: mapping[url] = rec_id return mapping
def update_doi(source, keys, outfile=None): if source.split(".")[-1] == "ds": # This generates report dot_paths = [".eprint_id", ".doi", ".related_url"] labels = ["eprint_id", "doi", "related_url"] all_metadata = get_records(dot_paths, "doi", source, keys, labels) for metadata in all_metadata: update = decide_doi_update(metadata) if update: outfile.writerow(update) else: for eprint_id in progressbar(keys, redirect_stdout=True): print(eprint_id) meta = get_eprint(source, eprint_id) # Ignore errors where the record doesn't exist if meta != None: update = decide_doi_update(meta) if update: url = source + "/rest/eprint/" + str(eprint_id) + "/doi.txt" headers = {"content-type": "text/plain"} doi = update[1].replace("\u200b", "") # Handle invisible charaters in ASM DOIs response = requests.put(url, data=doi, headers=headers) print(response)
def special_characters(source, keys, outfile=None): replacements = { "_0": "₀", "_1": "₁", "_2": "₂", "_3": "₃", "_4": "₄", "_5": "₅", "_6": "₆", "_7": "₇", "_8": "₈", "_9": "₉", "_+": "₊", "_-": "₋", "_a": "ₐ", "_e": "ₑ", "_o": "ₒ", "_x": "ₓ", "^0": "⁰", "^1": "¹", "^2": "²", "^3": "³", "^4": "⁴", "^5": "⁵", "^6": "⁶", "^7": "⁷", "^8": "⁸", "^9": "⁹", "^+": "⁺", "^-": "⁻", "^n": "ⁿ", "^i": "ⁱ", "’": "'", "“": '"', "”": '"', } if source.split(".")[-1] == "ds": dot_paths = [".eprint_id", ".title", ".abstract"] labels = ["eprint_id", "title", "abstract"] all_metadata = get_records(dot_paths, "official", source, keys, labels) outfile.writerow( [ "eprints_id", # "Current Title", "Updated Title", # "Current Abstract", "Updated Abstract", ] ) for meta in all_metadata: eprint_id = meta["eprint_id"] newtitle = replace_character(meta, "title", replacements) if "abstract" in meta: newabstract = replace_character(meta, "abstract", replacements) else: newabstract = None if outfile: if newtitle or newabstract: row = [eprint_id] if newtitle: row += [newtitle] # [meta["title"], newtitle] else: row += [" ", " "] if newabstract: row += [newabstract] # [meta["abstract"], newabstract] outfile.writerow(row)
def release_files(source, base_url, outfile=None): if source.split(".")[-1] == "ds": # This generates report dot_paths = [ ".eprint_id", ".documents", ".date", ".eprint_status", ".creators.items[0].name.family", ".thesis_type", ".full_text_status", ] labels = [ "eprint_id", "documents", "date", "status", "family", "type", "full_text", ] keys = dataset.keys(source) all_metadata = get_records(dot_paths, "official", source, keys, labels) all_metadata.sort(key=lambda all_metadata: all_metadata["family"]) all_metadata.sort(key=lambda all_metadata: all_metadata["date"]) for meta in all_metadata: year = meta["date"].split("-")[0] if is_in_range("2004-2005", year): if thesis_match(meta): files = [] fnames = [] count = 0 for document in meta["documents"]: count = count + 1 if document["security"] == "validuser": files.append(count) fnames.append(document["main"]) if len(files) > 0: eprint_id = meta["eprint_id"] print(eprint_id) outfile.writerow( [ year, meta["family"], eprint_id, meta["status"], meta["full_text"], files, fnames, ] ) mixed = False for filen in files: new = "public" # Doc status url = ( base_url + "/rest/eprint/" + str(eprint_id) + "/full_text_status.txt" ) response = requests.get(url) eprint_status = response.text if eprint_status == "restricted": response = requests.put(url, data=new, headers=headers) print(response) elif eprint_status == "mixed": print("mixed, skipping") mixed = True elif eprint_status != "public": print(eprint_status) print(url) exit() url = ( base_url + "/rest/eprint/" + str(eprint_id) + "/documents/" + str(filen) + "/security.txt" ) headers = {"content-type": "text/plain"} response = requests.get(url) live_status = response.text if not mixed: if live_status == "validuser": response = requests.put( url, data=new, headers=headers ) print(response) elif live_status != "public": print(live_status) print(url) exit()
def add_thesis_doi(data_collection, thesis_collection, token, production=True): """Add in theis DOI to CaltechDATA records""" # Search across CaltechTHESIS DOIs dot_paths = ["._Key", ".doi", ".official_url", ".related_url"] labels = ["eprint_id", "doi", "official_url", "related_url"] keys = dataset.keys(thesis_collection) all_metadata = get_records(dot_paths, "dois", thesis_collection, keys, labels) dois = [] for metadata in progressbar(all_metadata, redirect_stdout=True): if "doi" in metadata: record_doi = metadata["doi"].strip() if "related_url" in metadata and "items" in metadata["related_url"]: items = metadata["related_url"]["items"] for item in items: if "url" in item: url = item["url"].strip() if "type" in item: itype = item["type"].strip().lower() if itype == "doi": if idutils.is_doi(url): doi = "10." + url.split("10.")[1] prefix = doi.split("/")[0] if prefix == "10.22002": dois.append([doi, record_doi]) else: print("Ignoring non-DOI") print(metadata["eprint_id"]) print(url.split("10.")) for doi_link in dois: cd_doi = doi_link[0] thesis_doi = doi_link[1] print("Checking " + cd_doi) if "D1" in cd_doi: record_number = cd_doi.split("D1.")[1] if "d1" in cd_doi: record_number = cd_doi.split("d1.")[1] record, err = dataset.read(data_collection, record_number) if err != "": print(err) exit() done = False if "relatedIdentifiers" in record: for idv in record["relatedIdentifiers"]: identifier = idv["relatedIdentifier"] if identifier == thesis_doi: done = True if done == False: identifiers = record["relatedIdentifiers"] identifiers.append( { "relatedIdentifier": thesis_doi, "relatedIdentifierType": "DOI", "relationType": "IsSupplementTo", } ) new_metadata = {"relatedIdentifiers": identifiers} else: new_metadata = { "relatedIdentifiers": [ { "relatedIdentifier": thesis_doi, "relatedIdentifierType": "DOI", "relationType": "IsSupplementTo", } ] } if done == False: print("Adding " + thesis_doi + " to " + cd_doi) response = caltechdata_edit( token, record_number, new_metadata, {}, {}, True ) print(response)
if err != "": print(err) people_list = dataset.keys(import_coll) people = [] for p in people_list: record, err = dataset.read(import_coll, p) people.append(record) # Profiles collection from feeds profile_ds = "profiles.ds" keys = dataset.keys(profile_ds) labels = ["orcid", "creator_id"] dot_paths = [".orcid", ".creator_id"] all_metadata = get_records(dot_paths, "profile", profile_ds, keys, labels) for profile in all_metadata: if "creator_id" in profile: idv = profile["creator_id"] else: print("ERROR", profile) for person in people: if person["Authors_ID"] != "": if person["Authors_ID"] == idv: if person["ORCID"] == "": person["ORCID"] = profile["orcid"] dataset.update(import_coll, person["CL_PEOPLE_ID"], person) print("Updated ", person["CL_PEOPLE_ID"]) elif person["ORCID"] != profile["orcid"]: print( "Inconsistent ORCIDS for ",
def agent_report(file_name, repo, aspace): dot_paths = [ "._Key", ".directory_info", ".ORCID", ".sort_name", ".ArchivesSpace_ID", ".family", ".given", ] labels = ["id", "directory_info", "orcid", "name", "as", "family", "given"] source = get_caltechfeed("people") keys = dataset.keys(source) keys.remove("captured") all_metadata = get_records(dot_paths, "p_list", source, keys, labels) all_metadata.sort(key=lambda all_metadata: all_metadata["id"]) fname = file_name.split(".")[0] fcaltechpeople = fname + "_caltechpeople.csv" fmatched = fname + "_matched.csv" fnew_caltechpeople = fname + "_newcaltechpeople.csv" fnew_aspace = fname + "_newaspace.csv" caltechpeople = csv.writer(open(fcaltechpeople, "w")) matched = csv.writer(open(fmatched, "w")) new_caltechpeople = csv.writer(open(fnew_caltechpeople, "w")) new_aspace = csv.writer(open(fnew_aspace, "w")) to_match = {} gen_match = {} already_matched = {} aspace_url = "https://collections.archives.caltech.edu/agents/people/" feeds_url = "https://feeds.library.caltech.edu/people/" for metadata in all_metadata: if "as" in metadata: if metadata["as"] != "": already_matched[metadata["as"]] = metadata else: to_match[metadata["name"]] = metadata gen_match[metadata["family"]] = metadata print(f"{len(already_matched)} agents already in CaltechPEOPLE") print(f"Requesting agents") for agent in progressbar(aspace.agents): if agent.agent_type == "agent_person": primaty_name = agent.display_name.primary_name name = agent.display_name.sort_name published = agent.publish uid = int(agent.uri.split("/")[-1]) if uid not in already_matched: if name in to_match: person = to_match[name] matched.writerow([ person["name"], uid, aspace_url + str(uid), person["id"], feeds_url + person["id"], published, ]) to_match.pop(name) else: new_caltechpeople.writerow( [name, uid, aspace_url + str(uid), published]) else: metadata = already_matched[uid] caltechpeople.writerow([ metadata["name"], metadata["as"], aspace_url + str(metadata["as"]), metadata["id"], feeds_url + metadata["id"], published, ]) for name in to_match: new_aspace.writerow( [name, to_match[name]["id"], feeds_url + to_match[name]["id"]])