def get_rml_relationships(rml): """ relationship -> relationships """ result = {} rml_relationships = rml.findall(xmletree.prefixtag("rml", "relationship")) if rml_relationships is not None: relationships = [] for rml_relationship in rml_relationships: if rml_relationship is not None: # name -> agent.name name = get_rml_element_text(rml_relationship, "name") relationship = creator_service.formatted_name_to_creator(name, constants.REC_CLASS_PERSON, None) if relationship is None: relationship = {} relationship["agent"] = Person() # identifier -> agent.rec_id & agent.identifiers relationship["agent"].update(get_rml_identifiers(rml_relationship)) # relationType -> relation_type relationship.update(get_rml_element_text_and_set_key(rml_relationship, "relationType", "relation_type")) # descriptions -> descriptions relationship.update(get_rml_textlangs_and_set_key(rml_relationship, "description", "descriptions")) if relationship is not None: relationships.append(relationship) if relationships: result["relationships"] = relationships return result
def endnote_authors_to_creators(authors, creator_type="person", role="aut"): creators = [] for author in authors.splitlines(): creator = creator_service.formatted_name_to_creator(author, creator_type, role) if creator: creators.append(creator) return creators
def endnote_authors_to_creators(authors, creator_type="person", role="aut"): creators = [] for author in authors.splitlines(): creator = creator_service.formatted_name_to_creator( author, creator_type, role) if creator: creators.append(creator) return creators
def extract_creators(entry, key, role): #logging.debug("extract_creators") if key in entry.persons: #logging.debug("key: {}".format(key)) creators = [] authors = entry.persons[key] for author in authors: formatted_name = unicode(author).encode('utf-8') formatted_name = replace_special_characters(formatted_name) creator = creator_service.formatted_name_to_creator(formatted_name, constants.REC_CLASS_PERSON, role) creators.append(creator) return creators else: return None
def get_rml_teachings(rml): """ teaching -> teachings """ result = {} rml_teachings = rml.findall(xmletree.prefixtag("rml", "teaching")) if rml_teachings is not None: teachings = [] for rml_teaching in rml_teachings: if rml_teaching is not None: teaching = {} # dateBegin -> date_begin teaching.update(get_rml_element_text_and_set_key(rml_teaching, "dateBegin", "date_begin")) # dateEnd -> date_end teaching.update(get_rml_element_text_and_set_key(rml_teaching, "dateEnd", "date_end")) # description -> descriptions[i] teaching.update(get_rml_textlangs_and_set_key(rml_teaching, "description", "descriptions")) # level -> level teaching.update(get_rml_element_text_and_set_key(rml_teaching, "level", "level")) # title -> title teaching.update(get_rml_element_text_and_set_key(rml_teaching, "title", "title")) # creators # name -> creators[0].agent.name name = get_rml_element_text(rml_teaching, "name") creator = creator_service.formatted_name_to_creator(name, constants.REC_CLASS_ORGUNIT, "dgg") if creator is None: creator = Creator() creator["agent"] = Orgunit() creator["roles"] = "dgg" # identifiers -> creators[0].agent.rec_id or creators[0].agent.identifiers creator["agent"].update(get_rml_identifiers(rml_teaching)) if "name" in creator["agent"] or "rec_id" in creator["agent"] or "identifiers" in creator["agent"]: teaching["creators"] = [creator] if teaching is not None: teachings.append(teaching) if teachings: result["teachings"] = teachings return result
def get_rml_degrees(rml): """ degree -> degrees """ result = {} rml_degrees = rml.findall(xmletree.prefixtag("rml", "degree")) if rml_degrees is not None: degrees = [] for rml_degree in rml_degrees: if rml_degree is not None: degree = {} # dateBegin -> date_begin degree.update(get_rml_element_text_and_set_key(rml_degree, "dateBegin", "date_begin")) # dateEnd -> date_end degree.update(get_rml_element_text_and_set_key(rml_degree, "dateEnd", "date_end")) # description -> descriptions degree.update(get_rml_textlangs_and_set_key(rml_degree, "description", "descriptions")) # level -> level degree.update(get_rml_element_text_and_set_key(rml_degree, "level", "level")) # title -> title degree.update(get_rml_element_text_and_set_key(rml_degree, "title", "title")) # creators # name -> creators[0].agent.name name = get_rml_element_text(rml_degree, "name") creator = creator_service.formatted_name_to_creator(name, constants.REC_CLASS_ORGUNIT, "dgg") if creator is None: creator = Creator() creator["agent"] = Orgunit() creator["roles"] = ["dgg"] # identifiers -> creators[0].agent.rec_id or creators[0].agent.identifiers creator["agent"].update(get_rml_identifiers(rml_degree)) if "name" in creator["agent"] or "rec_id" in creator["agent"] or "identifiers" in creator["agent"]: degree["creators"] = [creator] if degree is not None: degrees.append(degree) if degrees: result["degrees"] = degrees return result
def get_rml_call(rml): """ call -> call """ result = {} rml_call = rml.find(xmletree.prefixtag("rml", "call")) if rml_call is not None: call = Call() # funding -> funding rml_funding = rml_call.find(xmletree.prefixtag("rml", "funding")) if rml_funding is not None: # name -> agent.name name = get_rml_element_text(rml_funding, "name") funding = creator_service.formatted_name_to_creator(name, constants.REC_CLASS_ORGUNIT, None) if funding is None: funding = Creator() funding["agent"] = Orgunit() # identifier -> agent.rec_id & agent.identifiers funding["agent"].update(get_rml_identifiers(rml_funding)) # programme -> programme funding.update(get_rml_element_text_and_set_key(rml_funding, "programme", "programme")) # scheme -> scheme funding.update(get_rml_element_text_and_set_key(rml_funding, "scheme", "scheme")) # contribution -> contribution funding.update(get_rml_money_and_set_key(rml_funding, "contribution", "contribution")) if funding: call["funding"] = funding # identifier -> rec_id call.update(get_rml_element_text_and_set_key(rml_call, "identifier", "rec_id")) # title -> title call.update(get_rml_element_text_and_set_key(rml_call, "title", "title")) # year -> date_issued call.update(get_rml_element_text_and_set_key(rml_call, "year", "date_issued")) if call: result["call"] = call return result
def get_rml_participants(rml): """ participant -> creators """ result = {} rml_participants = rml.findall(xmletree.prefixtag("rml", "participant")) if rml_participants is not None: creators = [] for rml_participant in rml_participants: if rml_participant is not None: creator_name = get_rml_element_text(rml_participant, "name") if creator_name: creator_rec_class = xmletree.get_element_attribute(rml_participant, "entityType") if creator_rec_class: creator_rec_class = creator_rec_class.title() creator = creator_service.formatted_name_to_creator(creator_name, creator_rec_class, None) if creator: creators.append(creator) if creators: result["creators"] = creators return result
def convert_creators(sum_doc, sum_authors_key, sum_affiliations_key, creator_type, role): creators = [] if sum_authors_key in sum_doc: sum_authors = sum_doc[sum_authors_key] if sum_authors: affiliations_dict = {} if sum_affiliations_key in sum_doc: sum_affiliations = sum_doc[sum_affiliations_key] if sum_affiliations: for index, affiliation in enumerate(sum_affiliations): affiliations_dict[index + 1] = affiliation for author in sum_authors: creator = Creator() if "surname" in author and "givenname" in author: person = Person() person.set_key_if_not_none("name_family", author["surname"]) person.set_key_if_not_none("name_given", author["givenname"]) creator["agent"] = person if role: creator["roles"] = [role] else: formatted_name = "" if "fullname" in author: formatted_name = author["fullname"] elif "name" in author: formatted_name = author["name"] creator = creator_service.formatted_name_to_creator(formatted_name, creator_type, role) if "sequence" in author and author["sequence"] in affiliations_dict: creator["affiliation"] = affiliations_dict[author["sequence"]] # todo : location dans le cas des DissertationSchool_xml if creator: creators.append(creator) return creators
def csv_dict_reader_to_metasjon(csv_row, input_format, source, rec_id_prefix): document = Document() if source: document["rec_source"] = source if input_format == constants.FORMAT_CSV_SITPOL: #logging.debug("csv_dict_reader_to_metasjon type(csv_row): {}".format(type(csv_row))) #print csv_row document["title"] = csv_row["title"] classifications_sitpol = [x.strip() for x in csv_row["classifications_sitpol"].split(";") if x.strip()] if classifications_sitpol: document["classifications_sitpol"] = classifications_sitpol classifications_ddc = [x.strip() for x in csv_row["classifications_ddc"].split(";") if x.strip()] if classifications_ddc: document["classifications_ddc"] = classifications_ddc formatted_names = [x.strip() for x in csv_row["creators@role=pbl"].split(";") if x.strip()] if formatted_names: #logging.debug("formatted_names: {}".format(formatted_names)) creators = [] for formatted_name in formatted_names: if formatted_name: creator = creator_service.formatted_name_to_creator(formatted_name, None, "pbl") if creator: creators.append(creator) if creators: document["creators"] = creators document["date_last_accessed"] = datetime.now().isoformat() document["descriptions"] = [{"language":"fr", "value":csv_row["descriptions@lang=fr"]}] keywords_fr = [x.strip() for x in csv_row["keywords@lang=fr"].split(";") if x.strip()] keywords_en = [x.strip() for x in csv_row["keywords@lang=en"].split(";") if x.strip()] keywords = {} if keywords_fr: keywords["fr"] = keywords_fr if keywords_en: keywords["en"] = keywords_en if keywords: document["keywords"] = keywords document["languages"] = [x.strip() for x in csv_row["languages"].split(";") if x.strip()] note = csv_row["notes@lang=fr"] if note: document["notes"] = note document["publication_countries"] = [x.strip() for x in csv_row["publication_countries"].split(";") if x.strip()] if "rec_created_user" in csv_row: document["rec_created_user"] = csv_row["rec_created_user"] document["rec_type_cerimes"] = csv_row["rec_type_cerimes"] specific_agents = [x.strip() for x in csv_row["specific_agents"].split(";") if x.strip()] if specific_agents: document["specific_agents"] = specific_agents document["specific_actor_type"] = csv_row["specific_actor_type"] document["target_audiences_cerimes"] = csv_row["target_audiences_cerimes"] document["url"] = csv_row["url"] document["rec_type"] = constants.DOC_TYPE_WEBENTITY document["webentity_type"] = csv_row["webentity_type"] elif input_format == constants.FORMAT_CSV_METAJSON: document["rec_type"] = "DatasetQuali" creators = [] if "Laboratoire d'inventaire" in csv_row: creators.append(creator_service.formatted_name_to_creator(csv_row["Laboratoire d'inventaire"], constants.REC_CLASS_ORGUNIT, "dpt")) document["title"] = csv_row["Titre de l'enquete"] if "Sujet(s) de l'enquete" in csv_row: document["keywords"] = [x.strip() for x in csv_row["Sujet(s) de l'enquete"].split("\n") if x.strip()] if "Nom Auteur 1" in csv_row: name_familly = csv_row["Nom Auteur 1"] name_given = affiliation = "" if "Prenom Auteur 1" in csv_row: name_given = csv_row["Prenom Auteur 1"] if "Affiliation Auteur 1" in csv_row: affiliation = csv_row["Affiliation Auteur 1"] document["creators"] = creators else: logging.error("Unknown input_format: {}".format(input_format)) logging.info(jsonbson.dumps_json(document, True)) return document
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix, only_first_record): document = None ris_type = None rec_type = None is_part_of_rec_type = None previous_key = None previous_value = None for line in txt_lines: if line: line = line.rstrip('\r\n') #logging.debug("line: {}".format(line)) # multi line management if previous_key: key = previous_key value = previous_value + line previous_key = None previous_value = None else: key = line[:2].strip() value = line[6:].strip() if value.endswith("/") and key not in ["Y1", "PY"]: #logging.debug("multi line") previous_key = key previous_value = value.rstrip('/') continue if key is None or len(key) == 0: # empty line -> continue #logging.debug("empty line") continue elif key == RIS_KEY_BEGIN: # record begin with document type -> create document # init document = Document() is_part_of_rec_type = None if source: document["rec_source"] = source ris_type = value rec_type = ris_document_type_to_metajson_document_type[ ris_type] document["rec_type"] = rec_type if ris_type in ris_document_type_to_metajson_document_is_part_of_type: is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[ ris_type] is_part_of = Document() is_part_of["rec_type"] = is_part_of_rec_type document["is_part_ofs"] = [is_part_of] elif key == RIS_KEY_END: # record end -> return the result # verify the is_part_ofs[0]["title"] if "is_part_ofs" in document and "title" not in document[ "is_part_ofs"][0] and "title_abbreviateds" in document[ "is_part_ofs"][0]: document["is_part_ofs"][0]["title"] = document[ "is_part_ofs"][0]["title_abbreviateds"][0]["title"] del document["is_part_ofs"][0]["title_abbreviateds"] logging.info("# RIS type: {}".format(ris_type)) metajson_service.pretty_print_document(document) yield document else: # process key value #logging.debug("key: {}; value: {}".format(key, value)) if key == "ID": document["rec_id"] = value elif key in [ "T1", "TI", "CT" ] or (key == "BT" and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Primary -> title document["title"] = value elif key in [ "JF", "JO" ] or (key == "BT" and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Secondary -> is_part_of["title"] document.add_is_part_of_title(value) elif key in ["JA", "J1", "J2", "T2"]: # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"] document.add_is_part_of_title_abbreviated(value) elif key == "T3": # Title Series document.add_series_title(value) elif key in ["A1", "AU"]: document.add_creator( creator_service.formatted_name_to_creator( value, None, "aut")) elif key in ["A2", "ED"]: if is_part_of_rec_type: document.add_is_part_of_creator( creator_service.formatted_name_to_creator( value, None, "edt")) else: document.add_creator( creator_service.formatted_name_to_creator( value, None, "edt")) elif key == "A3": document.add_series_creator( creator_service.formatted_name_to_creator( value, None, "aut")) elif key == "A4": document.add_creator( creator_service.formatted_name_to_creator( value, None, "ctb")) elif key in ["PY", "Y1", "DA"]: index_slash = value.find("/") if index_slash != -1: # YYYY/MM/DD/other info (like season) # todo document["date_issued"] = value.strip("/") else: document["date_issued"] = value elif key == "SP": document["part_page_begin"] = value elif key == "EP": document["part_page_end"] = value elif key == "VL": document["part_volume"] = value elif key in ["IS", "CP"]: document["part_issue"] = value elif key in ["AB", "N2"]: document["descriptions"] = [{ "language": "und", "value": value }] elif key == "N1": document["notes"] = [{"language": "und", "value": value}] elif key == "PB": document.add_item_to_key(value, "publishers") elif key == "CY": document.add_item_to_key(value, "publication_places") elif key == "RP": document["publication_status"] = value elif key == "ET": document["edition"] = value elif key == "UR": resource = Resource() resource["url"] = value document.add_item_to_key(resource, "resources") elif key == "AN": # Accession Number identifier = metajson_service.create_identifier( "accessionnumber", value) document.add_identifier(identifier) elif key == "CN": # Call Number identifier = metajson_service.create_identifier( "callnumber", value) document.add_identifier(identifier) elif key == "DO": # DOI identifier = metajson_service.create_identifier( "doi", value) document.add_identifier(identifier) elif key == "SN": # ISBN or ISSN ? id_type = None if rec_type in [ constants.DOC_TYPE_JOURNALARTICLE, constants.DOC_TYPE_MAGAZINEARTICLE, constants.DOC_TYPE_NEWSPAPERARTICLE, constants.DOC_TYPE_JOURNAL ]: id_type = "issn" else: id_type = "isbn" identifier = metajson_service.create_identifier( id_type, value) if is_part_of_rec_type is None: document.add_identifier(identifier) else: document["is_part_ofs"][0].add_identifier(identifier) elif key == "CA": document["caption"] = value elif key == "DB": # Name of Database -> rec_source ? document["rec_source"] = value elif key == "DP": # NDatabase Provider -> rec_source ? document["rec_source"] = value elif key == "KW": if "keywords" not in document: document["keywords"] = {"und": []} document["keywords"]["und"].append(value) else: logging.debug("Not managed key: {} with value: {}".format( key, value))
def endnotexml_record_to_metajson(record, source, rec_id_prefix): document = Document() # TODO # translated_creators: /contributors/translated-authors/author/style # auth_address: /auth-address/style # label: /label/style # custom1 # Extract endnote properties rec_id = record.find("rec-number").text endnote_type = record.find("ref-type").text rec_type = endnote_record_type_to_metajson_document_type[endnote_type] primary_creators = extract_creators(None, "aut", record, "./contributors/authors/author/style") secondary_creators = extract_creators( None, "pbd", record, "./contributors/secondary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: tertiary_creators = extract_creators( None, "pbd", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_THESIS: tertiary_creators = extract_creators( None, "ths", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: tertiary_creators = extract_creators( None, "pro", record, "./contributors/tertiary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: subsidiary_creators = extract_creators( None, "trl", record, "./contributors/subsidiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: subsidiary_creators = extract_creators( None, "act", record, "./contributors/subsidiary-authors/author/style") translated_creators = extract_creators( None, "trl", record, "./contributors/translated-authors/author/style") auth_address = extract_text(record, "./auth-address/style") title = extract_text(record, "./titles/title/style") title_secondary = extract_text(record, "./titles/secondary-title/style") title_tertiary = extract_text(record, "./titles/tertiary-title/style") title_alternative = extract_text(record, "./titles/alt-title/style") title_abbreviated = extract_text(record, "./titles/short-title/style") title_translated = extract_text(record, "./titles/translated-title/style") pages = extract_text(record, "./pages/style") part_volume = extract_text(record, "./volume/style") part_number = extract_text(record, "./number/style") extent_volumes = extract_text(record, "./num-vols/style") edition = extract_text(record, "./edition/style") part_section = extract_text(record, "./section/style") reprint_edition = extract_text(record, "./reprint-edition/style") keywords = extract_text(record, "./keywords/keyword/style") date_year = extract_text(record, "./dates/year/style") date_pub = extract_text(record, "./dates/pub-dates/date/style") publication_places_formatted = extract_text(record, "./pub-location/style") publishers_formatted = extract_text(record, "./publisher/style") orig_pub = extract_text(record, "./orig-pub/style") isbn_or_issn = extract_text(record, "./isbn/style") accessionnumber = extract_text(record, "./accession-num/style") callnumber = extract_text(record, "./call-num/style") if endnote_type == TYPE_WEB_PAGE: abstract = extract_text(record, "./pages/style") else: abstract = extract_text(record, "./abstract/style") label = extract_text(record, "./label/style") caption = extract_text(record, "./caption/style") note = extract_text(record, "./notes/style") reviewed_item = extract_text(record, "./reviewed-item/style") rec_type_description = extract_text(record, "./work-type/style") url = extract_text(record, "./urls/related-urls/url/style") custom1 = extract_text(record, "./custom1/style") custom2 = extract_text(record, "./custom2/style") custom3 = extract_text(record, "./custom3/style") custom4 = extract_text(record, "./custom4/style") custom5 = extract_text(record, "./custom5/style") custom6 = extract_text(record, "./custom6/style") custom7 = extract_text(record, "./custom7/style") doi = extract_text(record, "./electronic-resource-num/style") remote_database_name = extract_text(record, "./remote-database-name/style") remote_database_provider = extract_text( record, "./remote-database-provider/style") research_notes = extract_text(record, "./research-notes/style") language = extract_text(record, "./language/style") access_date = extract_text(record, "./access-date/style") # rec_id, rec_source document["rec_id"] = rec_id if source: document["rec_source"] = source # publishers_formatted, publication_places_formatted publishers = None publication_places = None if publishers_formatted: publishers = publishers_formatted.split("\r") if publication_places_formatted: publication_places = publication_places_formatted.split("\r") # type, is_part_of.type, is_part_of.is_part_of.type try: is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[ endnote_type] except: is_part_of_type = None is_part_of_is_part_of_type = None if title_secondary is not None: if endnote_type == TYPE_FIGURE: # how to determine the is_part_of type ? # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart if part_volume is not None or part_number is not None: is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL else: if title_translated is not None: is_part_of_type = constants.DOC_TYPE_BOOKPART is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK else: is_part_of_type = constants.DOC_TYPE_BOOK elif endnote_type == TYPE_FILM_OR_BROADCAST: rec_type = constants.DOC_TYPE_VIDEOPART is_part_of_type = constants.DOC_TYPE_VIDEORECORDING document["rec_type"] = rec_type document.set_key_if_not_none("rec_type_description", rec_type_description) # issn or isbn ? if is_part_of_type in [ constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER, constants.DOC_TYPE_MAGAZINE ]: isbn_or_issn_type = "issn" else: isbn_or_issn_type = "isbn" # is_part_of, is_part_of.is_part_of if is_part_of_type is not None and title_secondary: is_part_of = Document() is_part_of.set_key_if_not_none("rec_type", is_part_of_type) is_part_of.set_key_if_not_none("title", title_secondary) if is_part_of_is_part_of_type is not None: # is_part_of in case of is_part_of.is_part_of # creators with role aut is_part_of.add_creators( creator_service.change_contibutors_role( secondary_creators, "aut")) # is_part_of.is_part_of is_part_of_is_part_of = Document() is_part_of_is_part_of.set_key_if_not_none( "rec_type", is_part_of_is_part_of_type) is_part_of_is_part_of.set_key_if_not_none("title", title_translated) # creators with role pbd is_part_of_is_part_of.add_creators( creator_service.change_contibutors_role( translated_creators, "pbd")) #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of_is_part_of.set_key_if_not_none("publishers", publishers) is_part_of_is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs") else: # is_part_of in case of no is_part_of.is_part_of # creators with role edt is_part_of.add_creators(secondary_creators) #is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of.set_key_if_not_none("publishers", publishers) is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] if "title" in is_part_of and is_part_of["title"]: document.add_items_to_key([is_part_of], "is_part_ofs") else: if isbn_or_issn: document["identifiers"] = [ metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn) ] if publishers: if endnote_type == TYPE_THESIS: document.add_creators([ creator_service.formatted_name_to_creator( publishers[0], "orgunit", "dgg") ]) elif endnote_type == TYPE_FILM_OR_BROADCAST: document.add_creators([ creator_service.formatted_name_to_creator( publishers[0], "orgunit", "dst") ]) else: document.set_key_if_not_none("publishers", publishers) document.set_key_if_not_none("publication_places", publication_places) # seriess[] if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: series = Document() series["rec_type"] = constants.DOC_TYPE_SERIES if endnote_type == TYPE_BOOK and title_secondary: series.set_key_if_not_none("title", title_secondary) series.add_creators(secondary_creators) series.set_key_if_not_none("part_volume", part_number) if endnote_type == TYPE_BOOK_SECTION and title_tertiary: series.set_key_if_not_none("title", title_tertiary) series.add_creators(tertiary_creators) series.set_key_if_not_none("part_volume", part_number) if "title" in series and len(series) > 2: document.add_items_to_key([series], "seriess") # originals[] if (reprint_edition or title_translated or orig_pub) and endnote_type in [ TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST ]: original_title = None original_is_part_of = None if reprint_edition: original_title = reprint_edition if title_translated and is_part_of_is_part_of_type is None: original_title = title_translated if orig_pub: if is_part_of_type is not None: original_is_part_of = Document() original_is_part_of["rec_type"] = is_part_of_type original_is_part_of["title"] = orig_pub else: original_title = orig_pub if original_title: original = Document() original["rec_type"] = rec_type original.set_key_if_not_none("title", original_title) original.add_item_to_key(original_is_part_of, "is_part_ofs") document.add_item_to_key(original, "originals") # is_review_ofs[] if reviewed_item and endnote_type in [ TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE ]: is_review_ofs = Document() is_review_ofs.set_key_if_not_none("title", reviewed_item) is_review_ofs.set_key_if_not_none("rec_type", "Book") document.add_items_to_key([is_review_ofs], "is_review_ofs") # descriptions[0].value if abstract: document["descriptions"] = [{"value": abstract, "language": "und"}] # archive if endnote_type == TYPE_FIGURE and remote_database_provider: archive = Document() archive["title"] = remote_database_provider document.add_items_to_key([archive], "archive") # caption document.set_key_if_not_none("caption", caption) # creators[] document.add_creators(primary_creators) if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]: document.add_creators(tertiary_creators) if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]: document.add_creators(subsidiary_creators) if custom4: document.add_creators( endnote_authors_to_creators(custom4, "person", "rev")) if endnote_type == TYPE_FIGURE and remote_database_name: document.add_creators( endnote_authors_to_creators(remote_database_name, None, "cph")) # edition if endnote_type in [ TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE ] and edition: document["edition"] = edition # extent_pages, extent_volumes if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages: document["extent_pages"] = pages.replace("p.", "").strip() if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes: document["extent_volumes"] = extent_volumes # date_issued, date_issued_first if date_year: date_issued = "" date_issued_first = "" orig_index_begin = date_year.find("[") orig_index_end = date_year.find("]") if orig_index_begin != -1 and orig_index_end != -1: date_issued_first = date_year[orig_index_begin + 1:orig_index_end] date_issued = date_year.replace("[" + date_issued_first + "]", "").strip() else: date_issued = date_year.strip() if "is_part_ofs" in document: if document["is_part_ofs"][0][ "rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0].set_key_if_not_none( "date_issued", date_issued) document["is_part_ofs"][0].set_key_if_not_none( "date_issued_first", date_issued_first) elif "is_part_ofs" in document["is_part_ofs"][0] and document[ "is_part_ofs"][0]["is_part_ofs"][0][ "rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0]["is_part_ofs"][ 0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0]["is_part_ofs"][ 0].set_key_if_not_none("date_issued_first", date_issued_first) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) # identifiers[] identifiers = [] if accessionnumber: identifiers.append( metajson_service.create_identifier("accessionnumber", accessionnumber)) if callnumber: identifiers.append( metajson_service.create_identifier("callnumber", callnumber)) if doi: identifiers.append(metajson_service.create_identifier("doi", doi)) if identifiers: document["identifiers"] = identifiers # language if language: rfc5646 = language_service.convert_unknown_format_to_rfc5646(language) if rfc5646: document["languages"] = [rfc5646] # note if endnote_import_note and note: document.set_key_with_value_type_in_list("notes", note, "general") if endnote_import_research_note and research_notes: document.set_key_with_value_type_in_list("notes", research_notes, "user") # part_page_begin & part_page_end if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE ] and pages: hyphen_index = pages.find("-") if hyphen_index == -1: document["part_page_begin"] = pages.replace("p.", "").strip() else: document["part_page_begin"] = pages[:hyphen_index].replace( "p.", "").strip() document["part_page_end"] = pages[hyphen_index + 1:].replace( "p.", "").strip() if endnote_type in [TYPE_JOURNAL_ARTICLE]: document.set_key_if_not_none("part_issue", part_number) elif endnote_type in [TYPE_FIGURE]: document.set_key_if_not_none("part_number", part_number) document.set_key_if_not_none("part_section", part_section) document.set_key_if_not_none("part_volume", part_volume) # resources[0] if url is not None: resource = Resource() resource["rec_type"] = "ResourceRemote" resource.set_key_if_not_none("url", url) if endnote_type == TYPE_WEB_PAGE: resource.set_key_if_not_none("date_last_accessed", part_number) else: resource.set_key_if_not_none("date_last_accessed", access_date) document["resources"] = [resource] # subjects[] if endnote_import_keywords and keywords: for keyword in keywords.split(): document.set_key_with_value_type_in_list("subjects", keyword, "topic") # title, title_alternative, title_abbreviated, title_translated document["title"] = title if title_alternative: document["title_alternatives"] = [{"title": title_alternative}] if title_abbreviated: document["title_abbreviateds"] = [{"title": title_abbreviated}] #logging.debug("# endnote_type: {}".format(endnote_type)) metajson_service.pretty_print_document(document) return document
def endnotexml_record_to_metajson(record, source, rec_id_prefix): document = Document() # TODO # translated_creators: /contributors/translated-authors/author/style # auth_address: /auth-address/style # label: /label/style # custom1 # Extract endnote properties rec_id = record.find("rec-number").text endnote_type = record.find("ref-type").text rec_type = endnote_record_type_to_metajson_document_type[endnote_type] primary_creators = extract_creators(None, "aut", record, "./contributors/authors/author/style") secondary_creators = extract_creators(None, "pbd", record, "./contributors/secondary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: tertiary_creators = extract_creators(None, "pbd", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_THESIS: tertiary_creators = extract_creators(None, "ths", record, "./contributors/tertiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: tertiary_creators = extract_creators(None, "pro", record, "./contributors/tertiary-authors/author/style") if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: subsidiary_creators = extract_creators(None, "trl", record, "./contributors/subsidiary-authors/author/style") elif endnote_type == TYPE_FILM_OR_BROADCAST: subsidiary_creators = extract_creators(None, "act", record, "./contributors/subsidiary-authors/author/style") translated_creators = extract_creators(None, "trl", record, "./contributors/translated-authors/author/style") auth_address = extract_text(record, "./auth-address/style") title = extract_text(record, "./titles/title/style") title_secondary = extract_text(record, "./titles/secondary-title/style") title_tertiary = extract_text(record, "./titles/tertiary-title/style") title_alternative = extract_text(record, "./titles/alt-title/style") title_abbreviated = extract_text(record, "./titles/short-title/style") title_translated = extract_text(record, "./titles/translated-title/style") pages = extract_text(record, "./pages/style") part_volume = extract_text(record, "./volume/style") part_number = extract_text(record, "./number/style") extent_volumes = extract_text(record, "./num-vols/style") edition = extract_text(record, "./edition/style") part_section = extract_text(record, "./section/style") reprint_edition = extract_text(record, "./reprint-edition/style") keywords = extract_text(record, "./keywords/keyword/style") date_year = extract_text(record, "./dates/year/style") date_pub = extract_text(record, "./dates/pub-dates/date/style") publication_places_formatted = extract_text(record, "./pub-location/style") publishers_formatted = extract_text(record, "./publisher/style") orig_pub = extract_text(record, "./orig-pub/style") isbn_or_issn = extract_text(record, "./isbn/style") accessionnumber = extract_text(record, "./accession-num/style") callnumber = extract_text(record, "./call-num/style") if endnote_type == TYPE_WEB_PAGE: abstract = extract_text(record, "./pages/style") else: abstract = extract_text(record, "./abstract/style") label = extract_text(record, "./label/style") caption = extract_text(record, "./caption/style") note = extract_text(record, "./notes/style") reviewed_item = extract_text(record, "./reviewed-item/style") rec_type_description = extract_text(record, "./work-type/style") url = extract_text(record, "./urls/related-urls/url/style") custom1 = extract_text(record, "./custom1/style") custom2 = extract_text(record, "./custom2/style") custom3 = extract_text(record, "./custom3/style") custom4 = extract_text(record, "./custom4/style") custom5 = extract_text(record, "./custom5/style") custom6 = extract_text(record, "./custom6/style") custom7 = extract_text(record, "./custom7/style") doi = extract_text(record, "./electronic-resource-num/style") remote_database_name = extract_text(record, "./remote-database-name/style") remote_database_provider = extract_text(record, "./remote-database-provider/style") research_notes = extract_text(record, "./research-notes/style") language = extract_text(record, "./language/style") access_date = extract_text(record, "./access-date/style") # rec_id, rec_source document["rec_id"] = rec_id if source: document["rec_source"] = source # publishers_formatted, publication_places_formatted publishers = None publication_places = None if publishers_formatted: publishers = publishers_formatted.split("\r") if publication_places_formatted: publication_places = publication_places_formatted.split("\r") # type, is_part_of.type, is_part_of.is_part_of.type try: is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[endnote_type] except: is_part_of_type = None is_part_of_is_part_of_type = None if title_secondary is not None: if endnote_type == TYPE_FIGURE: # how to determine the is_part_of type ? # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart if part_volume is not None or part_number is not None: is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL else: if title_translated is not None: is_part_of_type = constants.DOC_TYPE_BOOKPART is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK else: is_part_of_type = constants.DOC_TYPE_BOOK elif endnote_type == TYPE_FILM_OR_BROADCAST: rec_type = constants.DOC_TYPE_VIDEOPART is_part_of_type = constants.DOC_TYPE_VIDEORECORDING document["rec_type"] = rec_type document.set_key_if_not_none("rec_type_description", rec_type_description) # issn or isbn ? if is_part_of_type in [constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER, constants.DOC_TYPE_MAGAZINE]: isbn_or_issn_type = "issn" else: isbn_or_issn_type = "isbn" # is_part_of, is_part_of.is_part_of if is_part_of_type is not None and title_secondary: is_part_of = Document() is_part_of.set_key_if_not_none("rec_type", is_part_of_type) is_part_of.set_key_if_not_none("title", title_secondary) if is_part_of_is_part_of_type is not None: # is_part_of in case of is_part_of.is_part_of # creators with role aut is_part_of.add_creators(creator_service.change_contibutors_role(secondary_creators, "aut")) # is_part_of.is_part_of is_part_of_is_part_of = Document() is_part_of_is_part_of.set_key_if_not_none("rec_type", is_part_of_is_part_of_type) is_part_of_is_part_of.set_key_if_not_none("title", title_translated) # creators with role pbd is_part_of_is_part_of.add_creators(creator_service.change_contibutors_role(translated_creators, "pbd")) #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of_is_part_of.set_key_if_not_none("publishers", publishers) is_part_of_is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs") else: # is_part_of in case of no is_part_of.is_part_of # creators with role edt is_part_of.add_creators(secondary_creators) #is_part_of.set_key_if_not_none("date_issued",date_year) is_part_of.set_key_if_not_none("publishers", publishers) is_part_of.set_key_if_not_none("publication_places", publication_places) if isbn_or_issn: is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] if "title" in is_part_of and is_part_of["title"]: document.add_items_to_key([is_part_of], "is_part_ofs") else: if isbn_or_issn: document["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)] if publishers: if endnote_type == TYPE_THESIS: document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dgg")]) elif endnote_type == TYPE_FILM_OR_BROADCAST: document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dst")]) else: document.set_key_if_not_none("publishers", publishers) document.set_key_if_not_none("publication_places", publication_places) # seriess[] if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]: series = Document() series["rec_type"] = constants.DOC_TYPE_SERIES if endnote_type == TYPE_BOOK and title_secondary: series.set_key_if_not_none("title", title_secondary) series.add_creators(secondary_creators) series.set_key_if_not_none("part_volume", part_number) if endnote_type == TYPE_BOOK_SECTION and title_tertiary: series.set_key_if_not_none("title", title_tertiary) series.add_creators(tertiary_creators) series.set_key_if_not_none("part_volume", part_number) if "title" in series and len(series) > 2: document.add_items_to_key([series], "seriess") # originals[] if (reprint_edition or title_translated or orig_pub) and endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST]: original_title = None original_is_part_of = None if reprint_edition: original_title = reprint_edition if title_translated and is_part_of_is_part_of_type is None: original_title = title_translated if orig_pub: if is_part_of_type is not None: original_is_part_of = Document() original_is_part_of["rec_type"] = is_part_of_type original_is_part_of["title"] = orig_pub else: original_title = orig_pub if original_title: original = Document() original["rec_type"] = rec_type original.set_key_if_not_none("title", original_title) original.add_item_to_key(original_is_part_of, "is_part_ofs") document.add_item_to_key(original, "originals") # is_review_ofs[] if reviewed_item and endnote_type in [TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE]: is_review_ofs = Document() is_review_ofs.set_key_if_not_none("title", reviewed_item) is_review_ofs.set_key_if_not_none("rec_type", "Book") document.add_items_to_key([is_review_ofs], "is_review_ofs") # descriptions[0].value if abstract: document["descriptions"] = [{"value": abstract, "language": "und"}] # archive if endnote_type == TYPE_FIGURE and remote_database_provider: archive = Document() archive["title"] = remote_database_provider document.add_items_to_key([archive], "archive") # caption document.set_key_if_not_none("caption", caption) # creators[] document.add_creators(primary_creators) if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]: document.add_creators(tertiary_creators) if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]: document.add_creators(subsidiary_creators) if custom4: document.add_creators(endnote_authors_to_creators(custom4, "person", "rev")) if endnote_type == TYPE_FIGURE and remote_database_name: document.add_creators(endnote_authors_to_creators(remote_database_name, None, "cph")) # edition if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE] and edition: document["edition"] = edition # extent_pages, extent_volumes if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages: document["extent_pages"] = pages.replace("p.", "").strip() if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes: document["extent_volumes"] = extent_volumes # date_issued, date_issued_first if date_year: date_issued = "" date_issued_first = "" orig_index_begin = date_year.find("[") orig_index_end = date_year.find("]") if orig_index_begin != -1 and orig_index_end != -1: date_issued_first = date_year[orig_index_begin + 1: orig_index_end] date_issued = date_year.replace("[" + date_issued_first + "]", "").strip() else: date_issued = date_year.strip() if "is_part_ofs" in document: if document["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first) elif "is_part_ofs" in document["is_part_ofs"][0] and document["is_part_ofs"][0]["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK: document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued) document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) else: document.set_key_if_not_none("date_issued_first", date_issued_first) if rec_type in unpublished_types: document.set_key_if_not_none("date_created", date_issued) else: document.set_key_if_not_none("date_issued", date_issued) # identifiers[] identifiers = [] if accessionnumber: identifiers.append(metajson_service.create_identifier("accessionnumber", accessionnumber)) if callnumber: identifiers.append(metajson_service.create_identifier("callnumber", callnumber)) if doi: identifiers.append(metajson_service.create_identifier("doi", doi)) if identifiers: document["identifiers"] = identifiers # language if language: rfc5646 = language_service.convert_unknown_format_to_rfc5646(language) if rfc5646: document["languages"] = [rfc5646] # note if endnote_import_note and note: document.set_key_with_value_type_in_list("notes", note, "general") if endnote_import_research_note and research_notes: document.set_key_with_value_type_in_list("notes", research_notes, "user") # part_page_begin & part_page_end if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE] and pages: hyphen_index = pages.find("-") if hyphen_index == -1: document["part_page_begin"] = pages.replace("p.", "").strip() else: document["part_page_begin"] = pages[:hyphen_index].replace("p.", "").strip() document["part_page_end"] = pages[hyphen_index+1:].replace("p.", "").strip() if endnote_type in [TYPE_JOURNAL_ARTICLE]: document.set_key_if_not_none("part_issue", part_number) elif endnote_type in [TYPE_FIGURE]: document.set_key_if_not_none("part_number", part_number) document.set_key_if_not_none("part_section", part_section) document.set_key_if_not_none("part_volume", part_volume) # resources[0] if url is not None: resource = Resource() resource["rec_type"] = "ResourceRemote" resource.set_key_if_not_none("url", url) if endnote_type == TYPE_WEB_PAGE: resource.set_key_if_not_none("date_last_accessed", part_number) else: resource.set_key_if_not_none("date_last_accessed", access_date) document["resources"] = [resource] # subjects[] if endnote_import_keywords and keywords: for keyword in keywords.split(): document.set_key_with_value_type_in_list("subjects", keyword, "topic") # title, title_alternative, title_abbreviated, title_translated document["title"] = title if title_alternative: document["title_alternatives"] = [{"title": title_alternative}] if title_abbreviated: document["title_abbreviateds"] = [{"title": title_abbreviated}] #logging.debug("# endnote_type: {}".format(endnote_type)) metajson_service.pretty_print_document(document) return document
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix, only_first_record): document = None ris_type = None rec_type = None is_part_of_rec_type = None previous_key = None previous_value = None for line in txt_lines: if line: line = line.rstrip('\r\n') #logging.debug("line: {}".format(line)) # multi line management if previous_key: key = previous_key value = previous_value + line previous_key = None previous_value = None else: key = line[:2].strip() value = line[6:].strip() if value.endswith("/") and key not in ["Y1", "PY"]: #logging.debug("multi line") previous_key = key previous_value = value.rstrip('/') continue if key is None or len(key) == 0: # empty line -> continue #logging.debug("empty line") continue elif key == RIS_KEY_BEGIN: # record begin with document type -> create document # init document = Document() is_part_of_rec_type = None if source: document["rec_source"] = source ris_type = value rec_type = ris_document_type_to_metajson_document_type[ris_type] document["rec_type"] = rec_type if ris_type in ris_document_type_to_metajson_document_is_part_of_type: is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[ris_type] is_part_of = Document() is_part_of["rec_type"] = is_part_of_rec_type document["is_part_ofs"] = [is_part_of] elif key == RIS_KEY_END: # record end -> return the result # verify the is_part_ofs[0]["title"] if "is_part_ofs" in document and "title" not in document["is_part_ofs"][0] and "title_abbreviateds" in document["is_part_ofs"][0]: document["is_part_ofs"][0]["title"] = document["is_part_ofs"][0]["title_abbreviateds"][0]["title"] del document["is_part_ofs"][0]["title_abbreviateds"] logging.info("# RIS type: {}".format(ris_type)) metajson_service.pretty_print_document(document) yield document else: # process key value #logging.debug("key: {}; value: {}".format(key, value)) if key == "ID": document["rec_id"] = value elif key in ["T1", "TI", "CT"] or (key == "BT" and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Primary -> title document["title"] = value elif key in ["JF", "JO"] or (key == "BT" and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]): # Title Secondary -> is_part_of["title"] document.add_is_part_of_title(value) elif key in ["JA", "J1", "J2", "T2"]: # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"] document.add_is_part_of_title_abbreviated(value) elif key == "T3": # Title Series document.add_series_title(value) elif key in ["A1", "AU"]: document.add_creator(creator_service.formatted_name_to_creator(value, None, "aut")) elif key in ["A2", "ED"]: if is_part_of_rec_type: document.add_is_part_of_creator(creator_service.formatted_name_to_creator(value, None, "edt")) else: document.add_creator(creator_service.formatted_name_to_creator(value, None, "edt")) elif key == "A3": document.add_series_creator(creator_service.formatted_name_to_creator(value, None, "aut")) elif key == "A4": document.add_creator(creator_service.formatted_name_to_creator(value, None, "ctb")) elif key in ["PY", "Y1", "DA"]: index_slash = value.find("/") if index_slash != -1: # YYYY/MM/DD/other info (like season) # todo document["date_issued"] = value.strip("/") else: document["date_issued"] = value elif key == "SP": document["part_page_begin"] = value elif key == "EP": document["part_page_end"] = value elif key == "VL": document["part_volume"] = value elif key in ["IS", "CP"]: document["part_issue"] = value elif key in ["AB", "N2"]: document["descriptions"] = [{"language": "und", "value": value}] elif key == "N1": document["notes"] = [{"language": "und", "value": value}] elif key == "PB": document.add_item_to_key(value, "publishers") elif key == "CY": document.add_item_to_key(value, "publication_places") elif key == "RP": document["publication_status"] = value elif key == "ET": document["edition"] = value elif key == "UR": resource = Resource() resource["url"] = value document.add_item_to_key(resource, "resources") elif key == "AN": # Accession Number identifier = metajson_service.create_identifier("accessionnumber", value) document.add_identifier(identifier) elif key == "CN": # Call Number identifier = metajson_service.create_identifier("callnumber", value) document.add_identifier(identifier) elif key == "DO": # DOI identifier = metajson_service.create_identifier("doi", value) document.add_identifier(identifier) elif key == "SN": # ISBN or ISSN ? id_type = None if rec_type in [constants.DOC_TYPE_JOURNALARTICLE, constants.DOC_TYPE_MAGAZINEARTICLE, constants.DOC_TYPE_NEWSPAPERARTICLE, constants.DOC_TYPE_JOURNAL]: id_type = "issn" else: id_type = "isbn" identifier = metajson_service.create_identifier(id_type, value) if is_part_of_rec_type is None: document.add_identifier(identifier) else: document["is_part_ofs"][0].add_identifier(identifier) elif key == "CA": document["caption"] = value elif key == "DB": # Name of Database -> rec_source ? document["rec_source"] = value elif key == "DP": # NDatabase Provider -> rec_source ? document["rec_source"] = value elif key == "KW": if "keywords" not in document: document["keywords"] = {"und": []} document["keywords"]["und"].append(value) else: logging.debug("Not managed key: {} with value: {}".format(key, value))