def convert_contributors(sum_doc, sum_authors_key, sum_affiliations_key, contributor_type, role): contributors = [] if sum_authors_key in sum_doc: sum_authors = sum_doc[sum_authors_key] if sum_authors: affiliations_dict = {} if sum_affiliations_key in sum_doc: sum_affiliations = sum_doc[sum_affiliations_key] if sum_affiliations: for index, affiliation in enumerate(sum_affiliations): affiliations_dict[index + 1] = affiliation for author in sum_authors: contributor = Contributor() if "surname" in author and "givenname" in author: contributor.set_key_if_not_none("name_family", author["surname"]) contributor.set_key_if_not_none("name_given", author["givenname"]) contributor["type"] = "person" if role: contributor["role"] = role else: formatted_name = "" if "fullname" in author: formatted_name = author["fullname"] elif "name" in author: formatted_name = author["name"] contributor = contributor_service.convert_formatted_name_to_contributor(formatted_name, contributor_type, role) if "sequence" in author and author["sequence"] in affiliations_dict: contributor["affiliation"] = affiliations_dict[author["sequence"]] if contributor: contributors.append(contributor) return contributors
def convert_endnote_authors_to_contributors(authors, contributor_type="person", role="aut"): contributors = [] for author in authors.splitlines(): contributor = contributor_service.convert_formatted_name_to_contributor(author, contributor_type, role) if contributor: contributors.append(contributor) return contributors
def extract_contributors(entry): if "author" in entry.persons: contributors = [] authors = entry.persons["author"] for author in authors: formatted_name = unicode(author).encode('utf-8') formatted_name = formatted_name.replace("{", "").replace("}", "") contributor = contributor_service.convert_formatted_name_to_contributor(formatted_name, "person", "aut") contributors.append(contributor) return contributors else: return None
def convert_endnote_record_to_metajson_document(record, source): document = Document() # TODO # translated_contributors: /contributors/translated-authors/author/style # auth_address: /auth-address/style # label: /label/style # custom1 # Extract endnote properties rec_id = record.find("rec-number").text endnote_type = record.find("ref-type").text rec_type = endnote_record_type_to_metajson_document_type[endnote_type] primary_contributors = extract_contributors(None, "aut", record, "./contributors/authors/author/style") secondary_contributors = extract_contributors(None, "edt", record, "./contributors/secondary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: tertiary_contributors = extract_contributors(None, "edt", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_THESIS: tertiary_contributors = extract_contributors(None, "ths", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: tertiary_contributors = extract_contributors(None, "pro", record, "./contributors/tertiary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: subsidiary_contributors = extract_contributors(None, "trl", record, "./contributors/subsidiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: subsidiary_contributors = extract_contributors(None, "act", record, "./contributors/subsidiary-authors/author/style") translated_contributors = extract_contributors(None, "trl", record, "./contributors/translated-authors/author/style") auth_address = extract_text(record, "./auth-address/style") title = extract_text(record, "./titles/title/style") title_secondary = extract_text(record, "./titles/secondary-title/style") title_tertiary = extract_text(record, "./titles/tertiary-title/style") title_alternative = extract_text(record, "./titles/alt-title/style") title_abbreviated = extract_text(record, "./titles/short-title/style") title_translated = extract_text(record, "./titles/translated-title/style") pages = extract_text(record, "./pages/style") part_volume = extract_text(record, "./volume/style") part_number = extract_text(record, "./number/style") extent_volumes = extract_text(record, "./num-vols/style") edition = extract_text(record, "./edition/style") part_section = extract_text(record, "./section/style") reprint_edition = extract_text(record, "./reprint-edition/style") keywords = extract_text(record, "./keywords/keyword/style") date_year = extract_text(record, "./dates/year/style") date_pub = extract_text(record, "./dates/pub-dates/date/style") publisher_place = extract_text(record, "./pub-location/style") publisher = extract_text(record, "./publisher/style") orig_pub = extract_text(record, "./orig-pub/style") isbn_or_issn = extract_text(record, "./isbn/style") accessionnumber = extract_text(record, "./accession-num/style") callnumber = extract_text(record, "./call-num/style") if endnote_type == TYPE_WEB_PAGE: abstract = extract_text(record, "./pages/style") else: abstract = extract_text(record, "./abstract/style") label = extract_text(record, "./label/style") caption = extract_text(record, "./caption/style") note = extract_text(record, "./notes/style") reviewed_item = extract_text(record, "./reviewed-item/style") rec_type_description = extract_text(record, "./work-type/style") remote_url = extract_text(record, "./urls/related-urls/url/style") custom1 = extract_text(record, "./custom1/style") custom2 = extract_text(record, "./custom2/style") custom3 = extract_text(record, "./custom3/style") custom4 = extract_text(record, "./custom4/style") custom5 = extract_text(record, "./custom5/style") custom6 = extract_text(record, "./custom6/style") custom7 = extract_text(record, "./custom7/style") doi = extract_text(record, "./electronic-resource-num/style") remote_database_name = extract_text(record, "./remote-database-name/style") remote_database_provider = extract_text(record, "./remote-database-provider/style") research_notes = extract_text(record, "./research-notes/style") language = extract_text(record, "./language/style") access_date = extract_text(record, "./access-date/style") # rec_id, rec_source document["rec_id"] = rec_id document["rec_source"] = source # publisher, publisher_place if publisher: publisher = publisher.replace("\r", "; ") if publisher_place: publisher_place = publisher_place.replace("\r", "; ") # type, is_part_of.type, is_part_of.is_part_of.type try: is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[endnote_type] except: is_part_of_type = None is_part_of_is_part_of_type = None if title_secondary is not None: if endnote_type == TYPE_FIGURE: # how to determine the is_part_of type ? # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookChapter if part_volume is not None or part_number is not None: is_part_of_type = "Article" is_part_of_is_part_of_type = "Journal" else: if title_translated is not None: is_part_of_type = "BookPart" is_part_of_is_part_of_type = "Book" else: is_part_of_type = "Book" elif endnote_type == TYPE_FILM_OR_BROADCAST: rec_type = "VideoPart" is_part_of_type = "VideoRecording" document["rec_type"] = rec_type document.set_key_if_not_none("rec_type_description", rec_type_description) # issn or isbn ? if is_part_of_type in ["Journal", "Newspaper", "Article"]: isbn_or_issn_type = "issn" else: isbn_or_issn_type = "isbn" # is_part_of, is_part_of.is_part_of if is_part_of_type is not None: is_part_of = Document() is_part_of.set_key_if_not_none("rec_type", is_part_of_type) is_part_of.set_key_if_not_none("title", title_secondary) if is_part_of_is_part_of_type is not None: # is_part_of in case of is_part_of.is_part_of # contributors with role aut is_part_of.add_contributors(contributor_service.change_contibutors_role(secondary_contributors, "aut")) # is_part_of.is_part_of is_part_of_is_part_of = Document() is_part_of_is_part_of.set_key_if_not_none("rec_type", is_part_of_is_part_of_type) is_part_of_is_part_of.set_key_if_not_none("title", title_translated) # contributors with role edt is_part_of_is_part_of.add_contributors(contributor_service.change_contibutors_role(translated_contributors, "edt")) #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of_is_part_of.set_key_if_not_none("publisher", publisher) is_part_of_is_part_of.set_key_if_not_none("publisher_place", publisher_place) is_part_of_is_part_of.set_key_with_value_type_in_list("identifiers", isbn_or_issn, isbn_or_issn_type) is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_of") else: # is_part_of in case of no is_part_of.is_part_of # contributors with role edt is_part_of.add_contributors(secondary_contributors) #is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of.set_key_if_not_none("publisher", publisher) is_part_of.set_key_if_not_none("publisher_place", publisher_place) is_part_of.set_key_with_value_type_in_list("identifiers", isbn_or_issn, isbn_or_issn_type) document.add_items_to_key([is_part_of], "is_part_of") else: document.set_key_with_value_type_in_list("identifiers", isbn_or_issn, isbn_or_issn_type) if publisher: if endnote_type == TYPE_THESIS: document.add_contributors([contributor_service.convert_formatted_name_to_contributor(publisher, "orgunit", "dgg")]) elif endnote_type == TYPE_FILM_OR_BROADCAST: document.add_contributors([contributor_service.convert_formatted_name_to_contributor(publisher, "orgunit", "dst")]) else: document.set_key_if_not_none("publisher", publisher) document.set_key_if_not_none("publisher_place", publisher_place) # series[] if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: series = Document() if endnote_type == TYPE_BOOK and title_secondary: series.set_key_if_not_none("title", title_secondary) series.add_contributors(secondary_contributors) series.set_key_if_not_none("part_volume", part_number) if endnote_type == TYPE_BOOK_SECTION and title_tertiary: series.set_key_if_not_none("title", title_tertiary) series.add_contributors(tertiary_contributors) series.set_key_if_not_none("part_volume", part_number) if "title" in series and len(series) > 2: document.add_items_to_key([series], "series") # originals[] if (reprint_edition or orig_pub) and endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST]: original = Document() if reprint_edition: original_title = reprint_edition elif orig_pub: original_title = orig_pub original.set_key_if_not_none("title", original_title) original.set_key_if_not_none("rec_type", rec_type) document.add_items_to_key([original], "original") # review_of[] if reviewed_item and endnote_type in [TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE]: review_of = Document() review_of.set_key_if_not_none("title", reviewed_item) review_of.set_key_if_not_none("rec_type", "Book") document.add_items_to_key([review_of], "review_of") # abstracts[0].value if abstract: document["abstracts"] = [{"value": abstract}] # archive if endnote_type == TYPE_FIGURE and remote_database_provider: archive = Document() archive["title"] = remote_database_provider document.add_items_to_key([archive], "archive") # caption document.set_key_if_not_none("caption", caption) # contributors[] document.add_contributors(primary_contributors) if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]: document.add_contributors(tertiary_contributors) if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]: document.add_contributors(subsidiary_contributors) if custom4: document.add_contributors(convert_endnote_authors_to_contributors(custom4, "person", "rev")) if endnote_type == TYPE_FIGURE and remote_database_name: document.add_contributors(convert_endnote_authors_to_contributors(remote_database_name, None, "cph")) # edition if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE] and edition: document["edition"] = edition # extent_pages, extent_volumes if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages: document["extent_pages"] = pages.replace("p.", "").strip() if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes: document["extent_volumes"] = extent_volumes # date_issued, date_issued_first if date_year: date_issued = "" date_issued_first = "" orig_index_start = date_year.find("[") orig_index_end = date_year.find("]") if orig_index_start != -1 and orig_index_end != -1: date_issued_first = date_year[orig_index_start + 1: orig_index_end] date_issued = date_year.replace("[" + date_issued_first + "]", "").strip() else: date_issued = date_year.strip() if "is_part_of" in document: if document["is_part_of"][0]["rec_type"] == "Book": document["is_part_of"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_of"][0].set_key_if_not_none("date_issued_first", date_issued_first) elif "is_part_of" in document["is_part_of"][0] and document["is_part_of"][0]["is_part_of"][0]["rec_type"] == "Book": document["is_part_of"][0]["is_part_of"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_of"][0]["is_part_of"][0].set_key_if_not_none("date_issued_first", date_issued_first) else: document.set_key_if_not_none("date_issued_first", date_issued_first) document.set_key_if_not_none("date_issued", date_issued) else: document.set_key_if_not_none("date_issued_first", date_issued_first) document.set_key_if_not_none("date_issued", date_issued) # identifiers[] document.set_key_with_value_type_in_list("identifiers", accessionnumber, "accessionnumber") document.set_key_with_value_type_in_list("identifiers", callnumber, "callnumber") document.set_key_with_value_type_in_list("identifiers", doi, "doi") # language if language: iso639_2 = language_service.convert_unknown_format_to_iso639_2(language) if iso639_2: document["languages"] = [{"authority": "iso639-2b", "value": iso639_2}] # note if endnote_import_note and note: document.set_key_with_value_type_in_list("notes", note, "general") if endnote_import_research_note and research_notes: document.set_key_with_value_type_in_list("notes", research_notes, "user") # part_page_start & part_page_end if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE] and pages: hyphen_index = pages.find("-") if hyphen_index == -1: document["part_page_start"] = pages.replace("p.", "").strip() else: document["part_page_start"] = pages[:hyphen_index].replace("p.", "").strip() document["part_page_end"] = pages[hyphen_index+1:].replace("p.", "").strip() if endnote_type in [TYPE_JOURNAL_ARTICLE]: document.set_key_if_not_none("part_issue", part_number) elif endnote_type in [TYPE_FIGURE]: document.set_key_if_not_none("part_number", part_number) document.set_key_if_not_none("part_section", part_section) document.set_key_if_not_none("part_volume", part_volume) # resources[0] if remote_url is not None: resource = Resource() resource.set_key_if_not_none("remote_url", remote_url) if endnote_type == TYPE_WEB_PAGE: resource.set_key_if_not_none("date_last_accessed", part_number) else: resource.set_key_if_not_none("date_last_accessed", access_date) document["resources"] = [resource] # subjects[] if endnote_import_keywords and keywords: for keyword in keywords.split(): document.set_key_with_value_type_in_list("subjects", keyword, "topic") # title, title_alternative, title_abbreviated, title_translated document["title"] = title if title_alternative: document["title_alternative"] = [{"title": title_alternative}] if title_abbreviated: document["title_abbreviated"] = [{"title": title_abbreviated}] if not is_part_of_is_part_of_type and title_translated: # the title_translated is used for the is_part_of.is_part_of_type.title document["title_translated"] = [{"title": title_translated}] debug = True if debug: related_items_msg = "" if is_part_of_type: related_items_msg = "\tis_part_of: {} ".format(is_part_of_type) if is_part_of_is_part_of_type: related_items_msg += "\tis_part_of.is_part_of: {} ".format(is_part_of_is_part_of_type) print "# {}\t:\t{}\t:\t{}\t->\titem: {} {}".format(source, rec_id, endnote_type, rec_type, related_items_msg, title) return document