def openurl_xmletree_to_metajson_list(openurl_response, source, rec_id_prefix, only_first_record): documents = [] if openurl_response is not None: #logging.debug(type(openurl_response)) #logging.debug(openurl_response) # results openurl_results = openurl_response.find( xmletree.prefixtag("ssopenurl", "results")) if openurl_results is not None: # result openurl_result_list = openurl_results.findall( xmletree.prefixtag("ssopenurl", "result")) if openurl_result_list: for openurl_result in openurl_result_list: document = Document() if source: document["source"] = source # citation openurl_citation = openurl_result.find( xmletree.prefixtag("ssopenurl", "citation")) if openurl_citation is not None: # issn openurl_issn = openurl_citation.find( xmletree.prefixtag("ssopenurl", "issn")) if openurl_issn is not None: identifier_issn = Identifier() identifier_issn["id_type"] = "issn" identifier_issn["value"] = openurl_issn.text document.add_item_to_key(identifier_issn, "identifiers") # eissn openurl_eissn = openurl_citation.find( xmletree.prefixtag("ssopenurl", "eissn")) if openurl_eissn is not None: identifier_eissn = Identifier() identifier_eissn["id_type"] = "eissn" identifier_eissn["value"] = openurl_eissn.text document.add_item_to_key(identifier_eissn, "identifiers") # linkGroups openurl_linkgroups = openurl_result.find( xmletree.prefixtag("ssopenurl", "linkGroups")) if openurl_linkgroups is not None: # linkGroup openurl_linkgroup_list = openurl_linkgroups.findall( xmletree.prefixtag("ssopenurl", "linkGroup")) if openurl_linkgroup_list is not None: for openurl_linkgroup in openurl_linkgroup_list: service_name = None institution_name = None period_begin = None period_end = None url = None # holdingData openurl_holdingdata = openurl_linkgroup.find( xmletree.prefixtag("ssopenurl", "holdingData")) if openurl_holdingdata is not None: # institution_name openurl_providername = openurl_holdingdata.find( xmletree.prefixtag( "ssopenurl", "providerName")) if openurl_providername is not None: institution_name = openurl_providername.text # service_name openurl_databasename = openurl_holdingdata.find( xmletree.prefixtag( "ssopenurl", "databaseName")) if openurl_databasename is not None: service_name = openurl_databasename.text # normalizedData openurl_normalizeddata = openurl_holdingdata.find( xmletree.prefixtag( "ssopenurl", "normalizedData")) if openurl_normalizeddata is not None: # startDate openurl_startdate = openurl_normalizeddata.find( xmletree.prefixtag( "ssopenurl", "startDate")) if openurl_startdate is not None: period_begin = openurl_startdate.text # endDate openurl_enddate = openurl_normalizeddata.find( xmletree.prefixtag( "ssopenurl", "endDate")) if openurl_enddate is not None: period_end = openurl_enddate.text # url openurl_url_list = openurl_linkgroup.findall( xmletree.prefixtag("ssopenurl", "url")) if openurl_url_list is not None: for openurl_url in openurl_url_list: if openurl_url.get( "type") == "journal": url = openurl_url.text elif openurl_url.get( "type") == "source": url = openurl_url.text if url: resource = Resource() resource["rec_type"] = "ResourceRemote" resource["rec_state"] = "published" resource["relation_type"] = "eResource" resource[ "version_type"] = "publishedVersion" resource["access_rights"] = "closedAccess" resource["format_mimetype"] = "text/html" resource["url"] = url if service_name: resource["service_name"] = service_name if institution_name: resource[ "institution_name"] = institution_name if period_begin: resource["period_begin"] = period_begin if period_end: resource["period_end"] = period_end document.add_item_to_key( resource, "resources") documents.append(document) if only_first_record: break #logging.debug(jsonbson.dumps_json(documents)) return documents
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix, only_first_record): document = None ris_type = None rec_type = None is_part_of_rec_type = None previous_key = None previous_value = None for line in txt_lines: if line: line = line.rstrip('\r\n') #logging.debug("line: {}".format(line)) # multi line management if previous_key: key = previous_key value = previous_value + line previous_key = None previous_value = None else: key = line[:2].strip() value = line[6:].strip() if value.endswith("/") and key not in ["Y1", "PY"]: #logging.debug("multi line") previous_key = key previous_value = value.rstrip('/') continue if key is None or len(key) == 0: # empty line -> continue #logging.debug("empty line") continue elif key == RIS_KEY_BEGIN: # record begin with document type -> create document # init document = Document() is_part_of_rec_type = None if source: document["rec_source"] = source ris_type = value rec_type = ris_document_type_to_metajson_document_type[ ris_type] document["rec_type"] = rec_type if ris_type in ris_document_type_to_metajson_document_is_part_of_type: is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[ ris_type] is_part_of = Document() is_part_of["rec_type"] = is_part_of_rec_type document["is_part_ofs"] = [is_part_of] elif key == RIS_KEY_END: # record end -> return the result # verify the is_part_ofs[0]["title"] if "is_part_ofs" in document and "title" not in document[ "is_part_ofs"][0] and "title_abbreviateds" in document[ "is_part_ofs"][0]: document["is_part_ofs"][0]["title"] = document[ "is_part_ofs"][0]["title_abbreviateds"][0]["title"] del document["is_part_ofs"][0]["title_abbreviateds"] logging.info("# RIS type: {}".format(ris_type)) metajson_service.pretty_print_document(document) yield document else: # process key value #logging.debug("key: {}; value: {}".format(key, value)) if key == "ID": document["rec_id"] = value elif key in [ "T1", "TI", "CT" ] or (key == "BT" and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Primary -> title document["title"] = value elif key in [ "JF", "JO" ] or (key == "BT" and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Secondary -> is_part_of["title"] document.add_is_part_of_title(value) elif key in ["JA", "J1", "J2", "T2"]: # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"] document.add_is_part_of_title_abbreviated(value) elif key == "T3": # Title Series document.add_series_title(value) elif key in ["A1", "AU"]: document.add_creator( creator_service.formatted_name_to_creator( value, None, "aut")) elif key in ["A2", "ED"]: if is_part_of_rec_type: document.add_is_part_of_creator( creator_service.formatted_name_to_creator( value, None, "edt")) else: document.add_creator( creator_service.formatted_name_to_creator( value, None, "edt")) elif key == "A3": document.add_series_creator( creator_service.formatted_name_to_creator( value, None, "aut")) elif key == "A4": document.add_creator( creator_service.formatted_name_to_creator( value, None, "ctb")) elif key in ["PY", "Y1", "DA"]: index_slash = value.find("/") if index_slash != -1: # YYYY/MM/DD/other info (like season) # todo document["date_issued"] = value.strip("/") else: document["date_issued"] = value elif key == "SP": document["part_page_begin"] = value elif key == "EP": document["part_page_end"] = value elif key == "VL": document["part_volume"] = value elif key in ["IS", "CP"]: document["part_issue"] = value elif key in ["AB", "N2"]: document["descriptions"] = [{ "language": "und", "value": value }] elif key == "N1": document["notes"] = [{"language": "und", "value": value}] elif key == "PB": document.add_item_to_key(value, "publishers") elif key == "CY": document.add_item_to_key(value, "publication_places") elif key == "RP": document["publication_status"] = value elif key == "ET": document["edition"] = value elif key == "UR": resource = Resource() resource["url"] = value document.add_item_to_key(resource, "resources") elif key == "AN": # Accession Number identifier = metajson_service.create_identifier( "accessionnumber", value) document.add_identifier(identifier) elif key == "CN": # Call Number identifier = metajson_service.create_identifier( "callnumber", value) document.add_identifier(identifier) elif key == "DO": # DOI identifier = metajson_service.create_identifier( "doi", value) document.add_identifier(identifier) elif key == "SN": # ISBN or ISSN ? id_type = None if rec_type in [ constants.DOC_TYPE_JOURNALARTICLE, constants.DOC_TYPE_MAGAZINEARTICLE, constants.DOC_TYPE_NEWSPAPERARTICLE, constants.DOC_TYPE_JOURNAL ]: id_type = "issn" else: id_type = "isbn" identifier = metajson_service.create_identifier( id_type, value) if is_part_of_rec_type is None: document.add_identifier(identifier) else: document["is_part_ofs"][0].add_identifier(identifier) elif key == "CA": document["caption"] = value elif key == "DB": # Name of Database -> rec_source ? document["rec_source"] = value elif key == "DP": # NDatabase Provider -> rec_source ? document["rec_source"] = value elif key == "KW": if "keywords" not in document: document["keywords"] = {"und": []} document["keywords"]["und"].append(value) else: logging.debug("Not managed key: {} with value: {}".format( key, value))
def openurl_xmletree_to_metajson_list(openurl_response, source, rec_id_prefix, only_first_record): documents = [] if openurl_response is not None: #logging.debug(type(openurl_response)) #logging.debug(openurl_response) # results openurl_results = openurl_response.find(xmletree.prefixtag("ssopenurl", "results")) if openurl_results is not None: # result openurl_result_list = openurl_results.findall(xmletree.prefixtag("ssopenurl", "result")) if openurl_result_list: for openurl_result in openurl_result_list: document = Document() if source: document["source"] = source # citation openurl_citation = openurl_result.find(xmletree.prefixtag("ssopenurl", "citation")) if openurl_citation is not None: # issn openurl_issn = openurl_citation.find(xmletree.prefixtag("ssopenurl", "issn")) if openurl_issn is not None: identifier_issn = Identifier() identifier_issn["id_type"] = "issn" identifier_issn["value"] = openurl_issn.text document.add_item_to_key(identifier_issn, "identifiers") # eissn openurl_eissn = openurl_citation.find(xmletree.prefixtag("ssopenurl", "eissn")) if openurl_eissn is not None: identifier_eissn = Identifier() identifier_eissn["id_type"] = "eissn" identifier_eissn["value"] = openurl_eissn.text document.add_item_to_key(identifier_eissn, "identifiers") # linkGroups openurl_linkgroups = openurl_result.find(xmletree.prefixtag("ssopenurl", "linkGroups")) if openurl_linkgroups is not None: # linkGroup openurl_linkgroup_list = openurl_linkgroups.findall(xmletree.prefixtag("ssopenurl", "linkGroup")) if openurl_linkgroup_list is not None: for openurl_linkgroup in openurl_linkgroup_list: service_name = None institution_name = None period_begin = None period_end = None url = None # holdingData openurl_holdingdata = openurl_linkgroup.find(xmletree.prefixtag("ssopenurl", "holdingData")) if openurl_holdingdata is not None: # institution_name openurl_providername = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "providerName")) if openurl_providername is not None: institution_name = openurl_providername.text # service_name openurl_databasename = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "databaseName")) if openurl_databasename is not None: service_name = openurl_databasename.text # normalizedData openurl_normalizeddata = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "normalizedData")) if openurl_normalizeddata is not None: # startDate openurl_startdate = openurl_normalizeddata.find(xmletree.prefixtag("ssopenurl", "startDate")) if openurl_startdate is not None: period_begin = openurl_startdate.text # endDate openurl_enddate = openurl_normalizeddata.find(xmletree.prefixtag("ssopenurl", "endDate")) if openurl_enddate is not None: period_end = openurl_enddate.text # url openurl_url_list = openurl_linkgroup.findall(xmletree.prefixtag("ssopenurl", "url")) if openurl_url_list is not None: for openurl_url in openurl_url_list: if openurl_url.get("type") == "journal": url = openurl_url.text elif openurl_url.get("type") == "source": url = openurl_url.text if url: resource = Resource() resource["rec_type"] = "ResourceRemote" resource["rec_state"] = "published" resource["relation_type"] = "eResource" resource["version_type"] = "publishedVersion" resource["access_rights"] = "closedAccess" resource["format_mimetype"] = "text/html" resource["url"] = url if service_name: resource["service_name"] = service_name if institution_name: resource["institution_name"] = institution_name if period_begin: resource["period_begin"] = period_begin if period_end: resource["period_end"] = period_end document.add_item_to_key(resource, "resources") documents.append(document) if only_first_record: break #logging.debug(jsonbson.dumps_json(documents)) return documents
def endnotexml_record_to_metajson(record, source, rec_id_prefix): document = Document() # TODO # translated_creators: /contributors/translated-authors/author/style # auth_address: /auth-address/style # label: /label/style # custom1 # Extract endnote properties rec_id = record.find("rec-number").text endnote_type = record.find("ref-type").text rec_type = endnote_record_type_to_metajson_document_type[endnote_type] primary_creators = extract_creators(None, "aut", record, "./contributors/authors/author/style") secondary_creators = extract_creators( None, "pbd", record, "./contributors/secondary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: tertiary_creators = extract_creators( None, "pbd", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_THESIS: tertiary_creators = extract_creators( None, "ths", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: tertiary_creators = extract_creators( None, "pro", record, "./contributors/tertiary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: subsidiary_creators = extract_creators( None, "trl", record, "./contributors/subsidiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: subsidiary_creators = extract_creators( None, "act", record, "./contributors/subsidiary-authors/author/style") translated_creators = extract_creators( None, "trl", record, "./contributors/translated-authors/author/style") auth_address = extract_text(record, "./auth-address/style") title = extract_text(record, "./titles/title/style") title_secondary = extract_text(record, "./titles/secondary-title/style") title_tertiary = extract_text(record, "./titles/tertiary-title/style") title_alternative = extract_text(record, "./titles/alt-title/style") title_abbreviated = extract_text(record, "./titles/short-title/style") title_translated = extract_text(record, "./titles/translated-title/style") pages = extract_text(record, "./pages/style") part_volume = extract_text(record, "./volume/style") part_number = extract_text(record, "./number/style") extent_volumes = extract_text(record, "./num-vols/style") edition = extract_text(record, "./edition/style") part_section = extract_text(record, "./section/style") reprint_edition = extract_text(record, "./reprint-edition/style") keywords = extract_text(record, "./keywords/keyword/style") date_year = extract_text(record, "./dates/year/style") date_pub = extract_text(record, "./dates/pub-dates/date/style") publication_places_formatted = extract_text(record, "./pub-location/style") publishers_formatted = extract_text(record, "./publisher/style") orig_pub = extract_text(record, "./orig-pub/style") isbn_or_issn = extract_text(record, "./isbn/style") accessionnumber = extract_text(record, "./accession-num/style") callnumber = extract_text(record, "./call-num/style") if endnote_type == TYPE_WEB_PAGE: abstract = extract_text(record, "./pages/style") else: abstract = extract_text(record, "./abstract/style") label = extract_text(record, "./label/style") caption = extract_text(record, "./caption/style") note = extract_text(record, "./notes/style") reviewed_item = extract_text(record, "./reviewed-item/style") rec_type_description = extract_text(record, "./work-type/style") url = extract_text(record, "./urls/related-urls/url/style") custom1 = extract_text(record, "./custom1/style") custom2 = extract_text(record, "./custom2/style") custom3 = extract_text(record, "./custom3/style") custom4 = extract_text(record, "./custom4/style") custom5 = extract_text(record, "./custom5/style") custom6 = extract_text(record, "./custom6/style") custom7 = extract_text(record, "./custom7/style") doi = extract_text(record, "./electronic-resource-num/style") remote_database_name = extract_text(record, "./remote-database-name/style") remote_database_provider = extract_text( record, "./remote-database-provider/style") research_notes = extract_text(record, "./research-notes/style") language = extract_text(record, "./language/style") access_date = extract_text(record, "./access-date/style") # rec_id, rec_source document["rec_id"] = rec_id if source: document["rec_source"] = source # publishers_formatted, publication_places_formatted publishers = None publication_places = None if publishers_formatted: publishers = publishers_formatted.split("\r") if publication_places_formatted: publication_places = publication_places_formatted.split("\r") # type, is_part_of.type, is_part_of.is_part_of.type try: is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[ endnote_type] except: is_part_of_type = None is_part_of_is_part_of_type = None if title_secondary is not None: if endnote_type == TYPE_FIGURE: # how to determine the is_part_of type ? # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart if part_volume is not None or part_number is not None: is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL else: if title_translated is not None: is_part_of_type = constants.DOC_TYPE_BOOKPART is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK else: is_part_of_type = constants.DOC_TYPE_BOOK elif endnote_type == TYPE_FILM_OR_BROADCAST: rec_type = constants.DOC_TYPE_VIDEOPART is_part_of_type = constants.DOC_TYPE_VIDEORECORDING document["rec_type"] = rec_type document.set_key_if_not_none("rec_type_description", rec_type_description) # issn or isbn ? if is_part_of_type in [ constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER, constants.DOC_TYPE_MAGAZINE ]: isbn_or_issn_type = "issn" else: isbn_or_issn_type = "isbn" # is_part_of, is_part_of.is_part_of if is_part_of_type is not None and title_secondary: is_part_of = Document() is_part_of.set_key_if_not_none("rec_type", is_part_of_type) is_part_of.set_key_if_not_none("title", title_secondary) if is_part_of_is_part_of_type is not None: # is_part_of in case of is_part_of.is_part_of # creators with role aut is_part_of.add_creators( creator_service.change_contibutors_role( secondary_creators, "aut")) # is_part_of.is_part_of is_part_of_is_part_of = Document() is_part_of_is_part_of.set_key_if_not_none( "rec_type", is_part_of_is_part_of_type) is_part_of_is_part_of.set_key_if_not_none("title", title_translated) # creators with role pbd is_part_of_is_part_of.add_creators( creator_service.change_contibutors_role( translated_creators, "pbd")) #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of_is_part_of.set_key_if_not_none("publishers", publishers) is_part_of_is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs") else: # is_part_of in case of no is_part_of.is_part_of # creators with role edt is_part_of.add_creators(secondary_creators) #is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of.set_key_if_not_none("publishers", publishers) is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] if "title" in is_part_of and is_part_of["title"]: document.add_items_to_key([is_part_of], "is_part_ofs") else: if isbn_or_issn: document["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] if publishers: if endnote_type == TYPE_THESIS: document.add_creators([ creator_service.formatted_name_to_creator( publishers[0], "orgunit", "dgg") ]) elif endnote_type == TYPE_FILM_OR_BROADCAST: document.add_creators([ creator_service.formatted_name_to_creator( publishers[0], "orgunit", "dst") ]) else: document.set_key_if_not_none("publishers", publishers) document.set_key_if_not_none("publication_places", publication_places) # seriess[] if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: series = Document() series["rec_type"] = constants.DOC_TYPE_SERIES if endnote_type == TYPE_BOOK and title_secondary: series.set_key_if_not_none("title", title_secondary) series.add_creators(secondary_creators) series.set_key_if_not_none("part_volume", part_number) if endnote_type == TYPE_BOOK_SECTION and title_tertiary: series.set_key_if_not_none("title", title_tertiary) series.add_creators(tertiary_creators) series.set_key_if_not_none("part_volume", part_number) if "title" in series and len(series) > 2: document.add_items_to_key([series], "seriess") # originals[] if (reprint_edition or title_translated or orig_pub) and endnote_type in [ TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST ]: original_title = None original_is_part_of = None if reprint_edition: original_title = reprint_edition if title_translated and is_part_of_is_part_of_type is None: original_title = title_translated if orig_pub: if is_part_of_type is not None: original_is_part_of = Document() original_is_part_of["rec_type"] = is_part_of_type original_is_part_of["title"] = orig_pub else: original_title = orig_pub if original_title: original = Document() original["rec_type"] = rec_type original.set_key_if_not_none("title", original_title) original.add_item_to_key(original_is_part_of, "is_part_ofs") document.add_item_to_key(original, "originals") # is_review_ofs[] if reviewed_item and endnote_type in [ TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE ]: is_review_ofs = Document() is_review_ofs.set_key_if_not_none("title", reviewed_item) is_review_ofs.set_key_if_not_none("rec_type", "Book") document.add_items_to_key([is_review_ofs], "is_review_ofs") # descriptions[0].value if abstract: document["descriptions"] = [{"value": abstract, "language": "und"}] # archive if endnote_type == TYPE_FIGURE and remote_database_provider: archive = Document() archive["title"] = remote_database_provider document.add_items_to_key([archive], "archive") # caption document.set_key_if_not_none("caption", caption) # creators[] document.add_creators(primary_creators) if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]: document.add_creators(tertiary_creators) if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]: document.add_creators(subsidiary_creators) if custom4: document.add_creators( endnote_authors_to_creators(custom4, "person", "rev")) if endnote_type == TYPE_FIGURE and remote_database_name: document.add_creators( endnote_authors_to_creators(remote_database_name, None, "cph")) # edition if endnote_type in [ TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE ] and edition: document["edition"] = edition # extent_pages, extent_volumes if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages: document["extent_pages"] = pages.replace("p.", "").strip() if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes: document["extent_volumes"] = extent_volumes # date_issued, date_issued_first if date_year: date_issued = "" date_issued_first = "" orig_index_begin = date_year.find("[") orig_index_end = date_year.find("]") if orig_index_begin != -1 and orig_index_end != -1: date_issued_first = date_year[orig_index_begin + 1:orig_index_end] date_issued = date_year.replace("[" + date_issued_first + "]", "").strip() else: date_issued = date_year.strip() if "is_part_ofs" in document: if document["is_part_ofs"][0][ "rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0].set_key_if_not_none( "date_issued", date_issued) document["is_part_ofs"][0].set_key_if_not_none( "date_issued_first", date_issued_first) elif "is_part_ofs" in document["is_part_ofs"][0] and document[ "is_part_ofs"][0]["is_part_ofs"][0][ "rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0]["is_part_ofs"][ 0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0]["is_part_ofs"][ 0].set_key_if_not_none("date_issued_first", date_issued_first) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) # identifiers[] identifiers = [] if accessionnumber: identifiers.append( metajson_service.create_identifier("accessionnumber", accessionnumber)) if callnumber: identifiers.append( metajson_service.create_identifier("callnumber", callnumber)) if doi: identifiers.append(metajson_service.create_identifier("doi", doi)) if identifiers: document["identifiers"] = identifiers # language if language: rfc5646 = language_service.convert_unknown_format_to_rfc5646(language) if rfc5646: document["languages"] = [rfc5646] # note if endnote_import_note and note: document.set_key_with_value_type_in_list("notes", note, "general") if endnote_import_research_note and research_notes: document.set_key_with_value_type_in_list("notes", research_notes, "user") # part_page_begin & part_page_end if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE ] and pages: hyphen_index = pages.find("-") if hyphen_index == -1: document["part_page_begin"] = pages.replace("p.", "").strip() else: document["part_page_begin"] = pages[:hyphen_index].replace( "p.", "").strip() document["part_page_end"] = pages[hyphen_index + 1:].replace( "p.", "").strip() if endnote_type in [TYPE_JOURNAL_ARTICLE]: document.set_key_if_not_none("part_issue", part_number) elif endnote_type in [TYPE_FIGURE]: document.set_key_if_not_none("part_number", part_number) document.set_key_if_not_none("part_section", part_section) document.set_key_if_not_none("part_volume", part_volume) # resources[0] if url is not None: resource = Resource() resource["rec_type"] = "ResourceRemote" resource.set_key_if_not_none("url", url) if endnote_type == TYPE_WEB_PAGE: resource.set_key_if_not_none("date_last_accessed", part_number) else: resource.set_key_if_not_none("date_last_accessed", access_date) document["resources"] = [resource] # subjects[] if endnote_import_keywords and keywords: for keyword in keywords.split(): document.set_key_with_value_type_in_list("subjects", keyword, "topic") # title, title_alternative, title_abbreviated, title_translated document["title"] = title if title_alternative: document["title_alternatives"] = [{"title": title_alternative}] if title_abbreviated: document["title_abbreviateds"] = [{"title": title_abbreviated}] #logging.debug("# endnote_type: {}".format(endnote_type)) metajson_service.pretty_print_document(document) return document
def endnotexml_record_to_metajson(record, source, rec_id_prefix): document = Document() # TODO # translated_creators: /contributors/translated-authors/author/style # auth_address: /auth-address/style # label: /label/style # custom1 # Extract endnote properties rec_id = record.find("rec-number").text endnote_type = record.find("ref-type").text rec_type = endnote_record_type_to_metajson_document_type[endnote_type] primary_creators = extract_creators(None, "aut", record, "./contributors/authors/author/style") secondary_creators = extract_creators(None, "pbd", record, "./contributors/secondary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: tertiary_creators = extract_creators(None, "pbd", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_THESIS: tertiary_creators = extract_creators(None, "ths", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: tertiary_creators = extract_creators(None, "pro", record, "./contributors/tertiary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: subsidiary_creators = extract_creators(None, "trl", record, "./contributors/subsidiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: subsidiary_creators = extract_creators(None, "act", record, "./contributors/subsidiary-authors/author/style") translated_creators = extract_creators(None, "trl", record, "./contributors/translated-authors/author/style") auth_address = extract_text(record, "./auth-address/style") title = extract_text(record, "./titles/title/style") title_secondary = extract_text(record, "./titles/secondary-title/style") title_tertiary = extract_text(record, "./titles/tertiary-title/style") title_alternative = extract_text(record, "./titles/alt-title/style") title_abbreviated = extract_text(record, "./titles/short-title/style") title_translated = extract_text(record, "./titles/translated-title/style") pages = extract_text(record, "./pages/style") part_volume = extract_text(record, "./volume/style") part_number = extract_text(record, "./number/style") extent_volumes = extract_text(record, "./num-vols/style") edition = extract_text(record, "./edition/style") part_section = extract_text(record, "./section/style") reprint_edition = extract_text(record, "./reprint-edition/style") keywords = extract_text(record, "./keywords/keyword/style") date_year = extract_text(record, "./dates/year/style") date_pub = extract_text(record, "./dates/pub-dates/date/style") publication_places_formatted = extract_text(record, "./pub-location/style") publishers_formatted = extract_text(record, "./publisher/style") orig_pub = extract_text(record, "./orig-pub/style") isbn_or_issn = extract_text(record, "./isbn/style") accessionnumber = extract_text(record, "./accession-num/style") callnumber = extract_text(record, "./call-num/style") if endnote_type == TYPE_WEB_PAGE: abstract = extract_text(record, "./pages/style") else: abstract = extract_text(record, "./abstract/style") label = extract_text(record, "./label/style") caption = extract_text(record, "./caption/style") note = extract_text(record, "./notes/style") reviewed_item = extract_text(record, "./reviewed-item/style") rec_type_description = extract_text(record, "./work-type/style") url = extract_text(record, "./urls/related-urls/url/style") custom1 = extract_text(record, "./custom1/style") custom2 = extract_text(record, "./custom2/style") custom3 = extract_text(record, "./custom3/style") custom4 = extract_text(record, "./custom4/style") custom5 = extract_text(record, "./custom5/style") custom6 = extract_text(record, "./custom6/style") custom7 = extract_text(record, "./custom7/style") doi = extract_text(record, "./electronic-resource-num/style") remote_database_name = extract_text(record, "./remote-database-name/style") remote_database_provider = extract_text(record, "./remote-database-provider/style") research_notes = extract_text(record, "./research-notes/style") language = extract_text(record, "./language/style") access_date = extract_text(record, "./access-date/style") # rec_id, rec_source document["rec_id"] = rec_id if source: document["rec_source"] = source # publishers_formatted, publication_places_formatted publishers = None publication_places = None if publishers_formatted: publishers = publishers_formatted.split("\r") if publication_places_formatted: publication_places = publication_places_formatted.split("\r") # type, is_part_of.type, is_part_of.is_part_of.type try: is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[endnote_type] except: is_part_of_type = None is_part_of_is_part_of_type = None if title_secondary is not None: if endnote_type == TYPE_FIGURE: # how to determine the is_part_of type ? # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart if part_volume is not None or part_number is not None: is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL else: if title_translated is not None: is_part_of_type = constants.DOC_TYPE_BOOKPART is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK else: is_part_of_type = constants.DOC_TYPE_BOOK elif endnote_type == TYPE_FILM_OR_BROADCAST: rec_type = constants.DOC_TYPE_VIDEOPART is_part_of_type = constants.DOC_TYPE_VIDEORECORDING document["rec_type"] = rec_type document.set_key_if_not_none("rec_type_description", rec_type_description) # issn or isbn ? if is_part_of_type in [constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER, constants.DOC_TYPE_MAGAZINE]: isbn_or_issn_type = "issn" else: isbn_or_issn_type = "isbn" # is_part_of, is_part_of.is_part_of if is_part_of_type is not None and title_secondary: is_part_of = Document() is_part_of.set_key_if_not_none("rec_type", is_part_of_type) is_part_of.set_key_if_not_none("title", title_secondary) if is_part_of_is_part_of_type is not None: # is_part_of in case of is_part_of.is_part_of # creators with role aut is_part_of.add_creators(creator_service.change_contibutors_role(secondary_creators, "aut")) # is_part_of.is_part_of is_part_of_is_part_of = Document() is_part_of_is_part_of.set_key_if_not_none("rec_type", is_part_of_is_part_of_type) is_part_of_is_part_of.set_key_if_not_none("title", title_translated) # creators with role pbd is_part_of_is_part_of.add_creators(creator_service.change_contibutors_role(translated_creators, "pbd")) #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of_is_part_of.set_key_if_not_none("publishers", publishers) is_part_of_is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs") else: # is_part_of in case of no is_part_of.is_part_of # creators with role edt is_part_of.add_creators(secondary_creators) #is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of.set_key_if_not_none("publishers", publishers) is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] if "title" in is_part_of and is_part_of["title"]: document.add_items_to_key([is_part_of], "is_part_ofs") else: if isbn_or_issn: document["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] if publishers: if endnote_type == TYPE_THESIS: document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dgg")]) elif endnote_type == TYPE_FILM_OR_BROADCAST: document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dst")]) else: document.set_key_if_not_none("publishers", publishers) document.set_key_if_not_none("publication_places", publication_places) # seriess[] if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: series = Document() series["rec_type"] = constants.DOC_TYPE_SERIES if endnote_type == TYPE_BOOK and title_secondary: series.set_key_if_not_none("title", title_secondary) series.add_creators(secondary_creators) series.set_key_if_not_none("part_volume", part_number) if endnote_type == TYPE_BOOK_SECTION and title_tertiary: series.set_key_if_not_none("title", title_tertiary) series.add_creators(tertiary_creators) series.set_key_if_not_none("part_volume", part_number) if "title" in series and len(series) > 2: document.add_items_to_key([series], "seriess") # originals[] if (reprint_edition or title_translated or orig_pub) and endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST]: original_title = None original_is_part_of = None if reprint_edition: original_title = reprint_edition if title_translated and is_part_of_is_part_of_type is None: original_title = title_translated if orig_pub: if is_part_of_type is not None: original_is_part_of = Document() original_is_part_of["rec_type"] = is_part_of_type original_is_part_of["title"] = orig_pub else: original_title = orig_pub if original_title: original = Document() original["rec_type"] = rec_type original.set_key_if_not_none("title", original_title) original.add_item_to_key(original_is_part_of, "is_part_ofs") document.add_item_to_key(original, "originals") # is_review_ofs[] if reviewed_item and endnote_type in [TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE]: is_review_ofs = Document() is_review_ofs.set_key_if_not_none("title", reviewed_item) is_review_ofs.set_key_if_not_none("rec_type", "Book") document.add_items_to_key([is_review_ofs], "is_review_ofs") # descriptions[0].value if abstract: document["descriptions"] = [{"value": abstract, "language": "und"}] # archive if endnote_type == TYPE_FIGURE and remote_database_provider: archive = Document() archive["title"] = remote_database_provider document.add_items_to_key([archive], "archive") # caption document.set_key_if_not_none("caption", caption) # creators[] document.add_creators(primary_creators) if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]: document.add_creators(tertiary_creators) if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]: document.add_creators(subsidiary_creators) if custom4: document.add_creators(endnote_authors_to_creators(custom4, "person", "rev")) if endnote_type == TYPE_FIGURE and remote_database_name: document.add_creators(endnote_authors_to_creators(remote_database_name, None, "cph")) # edition if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE] and edition: document["edition"] = edition # extent_pages, extent_volumes if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages: document["extent_pages"] = pages.replace("p.", "").strip() if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes: document["extent_volumes"] = extent_volumes # date_issued, date_issued_first if date_year: date_issued = "" date_issued_first = "" orig_index_begin = date_year.find("[") orig_index_end = date_year.find("]") if orig_index_begin != -1 and orig_index_end != -1: date_issued_first = date_year[orig_index_begin + 1: orig_index_end] date_issued = date_year.replace("[" + date_issued_first + "]", "").strip() else: date_issued = date_year.strip() if "is_part_ofs" in document: if document["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first) elif "is_part_ofs" in document["is_part_ofs"][0] and document["is_part_ofs"][0]["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) # identifiers[] identifiers = [] if accessionnumber: identifiers.append(metajson_service.create_identifier("accessionnumber", accessionnumber)) if callnumber: identifiers.append(metajson_service.create_identifier("callnumber", callnumber)) if doi: identifiers.append(metajson_service.create_identifier("doi", doi)) if identifiers: document["identifiers"] = identifiers # language if language: rfc5646 = language_service.convert_unknown_format_to_rfc5646(language) if rfc5646: document["languages"] = [rfc5646] # note if endnote_import_note and note: document.set_key_with_value_type_in_list("notes", note, "general") if endnote_import_research_note and research_notes: document.set_key_with_value_type_in_list("notes", research_notes, "user") # part_page_begin & part_page_end if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE] and pages: hyphen_index = pages.find("-") if hyphen_index == -1: document["part_page_begin"] = pages.replace("p.", "").strip() else: document["part_page_begin"] = pages[:hyphen_index].replace("p.", "").strip() document["part_page_end"] = pages[hyphen_index+1:].replace("p.", "").strip() if endnote_type in [TYPE_JOURNAL_ARTICLE]: document.set_key_if_not_none("part_issue", part_number) elif endnote_type in [TYPE_FIGURE]: document.set_key_if_not_none("part_number", part_number) document.set_key_if_not_none("part_section", part_section) document.set_key_if_not_none("part_volume", part_volume) # resources[0] if url is not None: resource = Resource() resource["rec_type"] = "ResourceRemote" resource.set_key_if_not_none("url", url) if endnote_type == TYPE_WEB_PAGE: resource.set_key_if_not_none("date_last_accessed", part_number) else: resource.set_key_if_not_none("date_last_accessed", access_date) document["resources"] = [resource] # subjects[] if endnote_import_keywords and keywords: for keyword in keywords.split(): document.set_key_with_value_type_in_list("subjects", keyword, "topic") # title, title_alternative, title_abbreviated, title_translated document["title"] = title if title_alternative: document["title_alternatives"] = [{"title": title_alternative}] if title_abbreviated: document["title_abbreviateds"] = [{"title": title_abbreviated}] #logging.debug("# endnote_type: {}".format(endnote_type)) metajson_service.pretty_print_document(document) return document
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix, only_first_record): document = None ris_type = None rec_type = None is_part_of_rec_type = None previous_key = None previous_value = None for line in txt_lines: if line: line = line.rstrip('\r\n') #logging.debug("line: {}".format(line)) # multi line management if previous_key: key = previous_key value = previous_value + line previous_key = None previous_value = None else: key = line[:2].strip() value = line[6:].strip() if value.endswith("/") and key not in ["Y1", "PY"]: #logging.debug("multi line") previous_key = key previous_value = value.rstrip('/') continue if key is None or len(key) == 0: # empty line -> continue #logging.debug("empty line") continue elif key == RIS_KEY_BEGIN: # record begin with document type -> create document # init document = Document() is_part_of_rec_type = None if source: document["rec_source"] = source ris_type = value rec_type = ris_document_type_to_metajson_document_type[ris_type] document["rec_type"] = rec_type if ris_type in ris_document_type_to_metajson_document_is_part_of_type: is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[ris_type] is_part_of = Document() is_part_of["rec_type"] = is_part_of_rec_type document["is_part_ofs"] = [is_part_of] elif key == RIS_KEY_END: # record end -> return the result # verify the is_part_ofs[0]["title"] if "is_part_ofs" in document and "title" not in document["is_part_ofs"][0] and "title_abbreviateds" in document["is_part_ofs"][0]: document["is_part_ofs"][0]["title"] = document["is_part_ofs"][0]["title_abbreviateds"][0]["title"] del document["is_part_ofs"][0]["title_abbreviateds"] logging.info("# RIS type: {}".format(ris_type)) metajson_service.pretty_print_document(document) yield document else: # process key value #logging.debug("key: {}; value: {}".format(key, value)) if key == "ID": document["rec_id"] = value elif key in ["T1", "TI", "CT"] or (key == "BT" and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Primary -> title document["title"] = value elif key in ["JF", "JO"] or (key == "BT" and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Secondary -> is_part_of["title"] document.add_is_part_of_title(value) elif key in ["JA", "J1", "J2", "T2"]: # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"] document.add_is_part_of_title_abbreviated(value) elif key == "T3": # Title Series document.add_series_title(value) elif key in ["A1", "AU"]: document.add_creator(creator_service.formatted_name_to_creator(value, None, "aut")) elif key in ["A2", "ED"]: if is_part_of_rec_type: document.add_is_part_of_creator(creator_service.formatted_name_to_creator(value, None, "edt")) else: document.add_creator(creator_service.formatted_name_to_creator(value, None, "edt")) elif key == "A3": document.add_series_creator(creator_service.formatted_name_to_creator(value, None, "aut")) elif key == "A4": document.add_creator(creator_service.formatted_name_to_creator(value, None, "ctb")) elif key in ["PY", "Y1", "DA"]: index_slash = value.find("/") if index_slash != -1: # YYYY/MM/DD/other info (like season) # todo document["date_issued"] = value.strip("/") else: document["date_issued"] = value elif key == "SP": document["part_page_begin"] = value elif key == "EP": document["part_page_end"] = value elif key == "VL": document["part_volume"] = value elif key in ["IS", "CP"]: document["part_issue"] = value elif key in ["AB", "N2"]: document["descriptions"] = [{"language": "und", "value": value}] elif key == "N1": document["notes"] = [{"language": "und", "value": value}] elif key == "PB": document.add_item_to_key(value, "publishers") elif key == "CY": document.add_item_to_key(value, "publication_places") elif key == "RP": document["publication_status"] = value elif key == "ET": document["edition"] = value elif key == "UR": resource = Resource() resource["url"] = value document.add_item_to_key(resource, "resources") elif key == "AN": # Accession Number identifier = metajson_service.create_identifier("accessionnumber", value) document.add_identifier(identifier) elif key == "CN": # Call Number identifier = metajson_service.create_identifier("callnumber", value) document.add_identifier(identifier) elif key == "DO": # DOI identifier = metajson_service.create_identifier("doi", value) document.add_identifier(identifier) elif key == "SN": # ISBN or ISSN ? id_type = None if rec_type in [constants.DOC_TYPE_JOURNALARTICLE, constants.DOC_TYPE_MAGAZINEARTICLE, constants.DOC_TYPE_NEWSPAPERARTICLE, constants.DOC_TYPE_JOURNAL]: id_type = "issn" else: id_type = "isbn" identifier = metajson_service.create_identifier(id_type, value) if is_part_of_rec_type is None: document.add_identifier(identifier) else: document["is_part_ofs"][0].add_identifier(identifier) elif key == "CA": document["caption"] = value elif key == "DB": # Name of Database -> rec_source ? document["rec_source"] = value elif key == "DP": # NDatabase Provider -> rec_source ? document["rec_source"] = value elif key == "KW": if "keywords" not in document: document["keywords"] = {"und": []} document["keywords"]["und"].append(value) else: logging.debug("Not managed key: {} with value: {}".format(key, value))