Exemple #1
0
def biblfull_xmletree_to_metajson(biblfull, laboratories, projects, source):
    """ biblFull xmletree -> MetaJSON Document """
    if biblfull is None:
        return None

    document = Document()

    # titleStmt
    tei_titlestmt = biblfull.find(xmletree.prefixtag("tei", "titleStmt"))
    # editionStmt
    tei_editionstmt = biblfull.find(xmletree.prefixtag("tei", "editionStmt"))
    # extent
    tei_extent = biblfull.find(xmletree.prefixtag("tei", "extent"))
    # publicationStmt
    tei_publicationstmt = biblfull.find(
        xmletree.prefixtag("tei", "publicationStmt"))
    # seriesStmt
    tei_seriesstmt = biblfull.find(xmletree.prefixtag("tei", "seriesStmt"))
    # notesStmt
    tei_notesstmt = biblfull.find(xmletree.prefixtag("tei", "notesStmt"))
    # sourceDesc
    tei_sourcedescs = biblfull.findall(xmletree.prefixtag("tei", "sourceDesc"))
    # profileDesc
    tei_profiledesc = biblfull.find(xmletree.prefixtag("tei", "profileDesc"))
    tei_langusage = tei_profiledesc.find(xmletree.prefixtag(
        "tei", "langUsage"))
    tei_languages = tei_langusage.findall(xmletree.prefixtag(
        "tei", "language"))
    tei_textclass = tei_profiledesc.find(xmletree.prefixtag(
        "tei", "textClass"))
    tei_keywords = tei_textclass.findall(xmletree.prefixtag("tei", "keywords"))
    tei_classcodes = tei_textclass.findall(
        xmletree.prefixtag("tei", "classCode"))

    # language
    doc_language = None
    if tei_languages:
        languages = []
        for tei_language in tei_languages:
            language = tei_language.get("ident")
            languages.append(language)
        if languages:
            document["languages"] = languages
            doc_language = languages[0]

    # title
    document.update(get_tei_titles_to_metason(tei_titlestmt, doc_language))

    metajson_service.pretty_print_document(document)
    metajson_service.print_document(document)
    return document
Exemple #2
0
def ddi_xmletree_to_metajson(ddi_root, source, rec_id_prefix):
    """ DDI xmletree -> MetaJSON Document """
    if ddi_root is None:
        return None

    document = Document()

    document["rec_type"] = constants.DOC_TYPE_DATASETQUALI

    if source:
        document["rec_source"] = source

    # stdyDscr/citation/titlStmt/titl
    ddi_stdydscr = ddi_root.find(xmletree.prefixtag("ddi", "stdyDscr"))
    if ddi_stdydscr is not None:
        ddi_stdydscr_citation = ddi_stdydscr.find(
            xmletree.prefixtag("ddi", "citation"))
        ddi_stdydscr_citation_titlstmt = ddi_stdydscr_citation.find(
            xmletree.prefixtag("ddi", "titlStmt"))
        ddi_stdydscr_citation_titlstmt_titl = ddi_stdydscr_citation_titlstmt.find(
            xmletree.prefixtag("ddi", "titl"))
        if ddi_stdydscr_citation_titlstmt_titl is not None:
            document["title"] = ddi_stdydscr_citation_titlstmt_titl.text

    return document
Exemple #3
0
def load_dict(meta_dict):
    if "rec_class" not in meta_dict:
        return Common(meta_dict)
    elif meta_dict["rec_class"] == "Document":
        return Document(meta_dict)
    elif meta_dict["rec_class"] == "Person":
        return Person(meta_dict)
    elif meta_dict["rec_class"] == "Orgunit":
        return Orgunit(meta_dict)
    elif meta_dict["rec_class"] == "Project":
        return Project(meta_dict)
    elif meta_dict["rec_class"] == "Event":
        return Event(meta_dict)
    elif meta_dict["rec_class"] == "Family":
        return Family(meta_dict)
    elif meta_dict["rec_class"] == "Field":
        return Field(meta_dict)
    elif meta_dict["rec_class"] == "Resource":
        return Resource(meta_dict)
    elif meta_dict["rec_class"] == "Target":
        return Target(meta_dict)
    elif meta_dict["rec_class"] == "Type":
        return Type(meta_dict)
    elif meta_dict["rec_class"] == "Collection":
        return Collection(meta_dict)
    else:
        logging.debug(jsonbson.dumps_bson(meta_dict))
        logging.warning("Unknown rec_class: {O}".format(meta_dict["rec_class"]))
        return Common(meta_dict)
Exemple #4
0
def tikadict_to_metajson(tikadict, source):
    document = Document()
    resource = Resource()

    if source:
        document["source"] = source

    # document
    # dc:title -> document title
    if "dc:title" in tikadict:
        document["title"] = tikadict["dc:title"]
    # dcterms:created -> document date_issued ou resource rec_created_date ?
    if "dcterms:created" in tikadict:
        document["date_issued"] = tikadict["dcterms:created"]
    # xmpTPg:NPages -> document extent_pages
    if "xmpTPg:NPages" in tikadict:
        document["extent_pages"] = tikadict["xmpTPg:NPages"]

    # resource
    # dcterms:modified -> resource rec_modified_date ?
    if "dcterms:modified" in tikadict:
        resource["rec_modified_date"] = tikadict["dcterms:modified"]
    # producer -> resource processing_software_name
    if "producer" in tikadict:
        resource["processing_software_name"] = tikadict["producer"]
    # Content-Type -> resource format_mimetype
    if "Content-Type" in tikadict:
        resource["format_mimetype"] = tikadict["Content-Type"]

    document["resources"] = [resource]

    logging.debug(document)
    return document
def biblfull_xmletree_to_metajson(biblfull, laboratories, projects, source):
    """ biblFull xmletree -> MetaJSON Document """
    if biblfull is None:
        return None

    document = Document()

    # titleStmt
    tei_titlestmt = biblfull.find(xmletree.prefixtag("tei", "titleStmt"))
    # editionStmt
    tei_editionstmt = biblfull.find(xmletree.prefixtag("tei", "editionStmt"))
    # extent
    tei_extent = biblfull.find(xmletree.prefixtag("tei", "extent"))
    # publicationStmt
    tei_publicationstmt = biblfull.find(xmletree.prefixtag("tei", "publicationStmt"))
    # seriesStmt
    tei_seriesstmt = biblfull.find(xmletree.prefixtag("tei", "seriesStmt"))
    # notesStmt
    tei_notesstmt = biblfull.find(xmletree.prefixtag("tei", "notesStmt"))
    # sourceDesc
    tei_sourcedescs = biblfull.findall(xmletree.prefixtag("tei", "sourceDesc"))
    # profileDesc
    tei_profiledesc = biblfull.find(xmletree.prefixtag("tei", "profileDesc"))
    tei_langusage = tei_profiledesc.find(xmletree.prefixtag("tei", "langUsage"))
    tei_languages = tei_langusage.findall(xmletree.prefixtag("tei", "language"))
    tei_textclass = tei_profiledesc.find(xmletree.prefixtag("tei", "textClass"))
    tei_keywords = tei_textclass.findall(xmletree.prefixtag("tei", "keywords"))
    tei_classcodes = tei_textclass.findall(xmletree.prefixtag("tei", "classCode"))

    # language
    doc_language = None
    if tei_languages:
        languages = []
        for tei_language in tei_languages:
            language = tei_language.get("ident")
            languages.append(language)
        if languages:
            document["languages"] = languages
            doc_language = languages[0]

    # title
    document.update(get_tei_titles_to_metason(tei_titlestmt, doc_language))

    metajson_service.pretty_print_document(document)
    metajson_service.print_document(document)
    return document
Exemple #6
0
def hyphe_webentity_to_metajson(webentity, source):
    document = Document()

    # static
    document["rec_type"] = "WebEntity"
    if source:
        document["source"] = source

    # MetaJSON
    # id -> rec_id
    document["rec_id"] = webentity["id"]
    # creation_date -> rec_created_date
    #document["rec_created_date"] = date_service.parse_timestamp(webentity["creation_date"])
    # last_modification_date -> rec_modified_date
    #document["rec_modified_date"] = date_service.parse_timestamp(webentity["last_modification_date"])
    document["title"] = webentity["name"]

    # Hyphe specific
    hyphe = {}
    # id -> hyphe/webentity_id
    hyphe["webentity_id"] = webentity["id"]
    # crawling_status -> hyphe/crawling_status
    hyphe["crawling_status"] = webentity["crawling_status"]
    # indexing_status -> hyphe/indexing_status
    hyphe["indexing_status"] = webentity["indexing_status"]
    # status -> hyphe/status
    hyphe["status"] = webentity["status"]
    # lru_prefixes -> hyphe/lru_prefixes
    hyphe["lru_prefixes"] = webentity["lru_prefixes"]
    # startpages -> hyphe/startpages
    hyphe["startpages"] = webentity["startpages"]

    # todo : tags to metajson specific
    #if webentity["tags"]:
    #    hyphe["tags"] = webentity["tags"]

    document["hyphe"] = hyphe

    return document
Exemple #7
0
def mets_xmletree_to_metajson(mets, source, rec_id_prefix):
    document = Document()

    # source
    if source:
        document["rec_source"] = source

    # mets

    # metsHdr

    # dmdSec
    document["dmds"] = extract_dmdsecs(mets)

    # techMD

    # sourceMD

    # digiprovMD

    # fileGrp

    # structMap
    return document
Exemple #8
0
def openurl_xmletree_to_metajson_list(openurl_response, source, rec_id_prefix,
                                      only_first_record):
    documents = []
    if openurl_response is not None:
        #logging.debug(type(openurl_response))
        #logging.debug(openurl_response)
        # results
        openurl_results = openurl_response.find(
            xmletree.prefixtag("ssopenurl", "results"))
        if openurl_results is not None:
            # result
            openurl_result_list = openurl_results.findall(
                xmletree.prefixtag("ssopenurl", "result"))
            if openurl_result_list:
                for openurl_result in openurl_result_list:
                    document = Document()
                    if source:
                        document["source"] = source
                    # citation
                    openurl_citation = openurl_result.find(
                        xmletree.prefixtag("ssopenurl", "citation"))
                    if openurl_citation is not None:
                        # issn
                        openurl_issn = openurl_citation.find(
                            xmletree.prefixtag("ssopenurl", "issn"))
                        if openurl_issn is not None:
                            identifier_issn = Identifier()
                            identifier_issn["id_type"] = "issn"
                            identifier_issn["value"] = openurl_issn.text
                            document.add_item_to_key(identifier_issn,
                                                     "identifiers")
                        # eissn
                        openurl_eissn = openurl_citation.find(
                            xmletree.prefixtag("ssopenurl", "eissn"))
                        if openurl_eissn is not None:
                            identifier_eissn = Identifier()
                            identifier_eissn["id_type"] = "eissn"
                            identifier_eissn["value"] = openurl_eissn.text
                            document.add_item_to_key(identifier_eissn,
                                                     "identifiers")
                    # linkGroups
                    openurl_linkgroups = openurl_result.find(
                        xmletree.prefixtag("ssopenurl", "linkGroups"))
                    if openurl_linkgroups is not None:
                        # linkGroup
                        openurl_linkgroup_list = openurl_linkgroups.findall(
                            xmletree.prefixtag("ssopenurl", "linkGroup"))
                        if openurl_linkgroup_list is not None:
                            for openurl_linkgroup in openurl_linkgroup_list:
                                service_name = None
                                institution_name = None
                                period_begin = None
                                period_end = None
                                url = None
                                # holdingData
                                openurl_holdingdata = openurl_linkgroup.find(
                                    xmletree.prefixtag("ssopenurl",
                                                       "holdingData"))
                                if openurl_holdingdata is not None:
                                    # institution_name
                                    openurl_providername = openurl_holdingdata.find(
                                        xmletree.prefixtag(
                                            "ssopenurl", "providerName"))
                                    if openurl_providername is not None:
                                        institution_name = openurl_providername.text
                                    # service_name
                                    openurl_databasename = openurl_holdingdata.find(
                                        xmletree.prefixtag(
                                            "ssopenurl", "databaseName"))
                                    if openurl_databasename is not None:
                                        service_name = openurl_databasename.text
                                    # normalizedData
                                    openurl_normalizeddata = openurl_holdingdata.find(
                                        xmletree.prefixtag(
                                            "ssopenurl", "normalizedData"))
                                    if openurl_normalizeddata is not None:
                                        # startDate
                                        openurl_startdate = openurl_normalizeddata.find(
                                            xmletree.prefixtag(
                                                "ssopenurl", "startDate"))
                                        if openurl_startdate is not None:
                                            period_begin = openurl_startdate.text
                                        # endDate
                                        openurl_enddate = openurl_normalizeddata.find(
                                            xmletree.prefixtag(
                                                "ssopenurl", "endDate"))
                                        if openurl_enddate is not None:
                                            period_end = openurl_enddate.text
                                # url
                                openurl_url_list = openurl_linkgroup.findall(
                                    xmletree.prefixtag("ssopenurl", "url"))
                                if openurl_url_list is not None:
                                    for openurl_url in openurl_url_list:
                                        if openurl_url.get(
                                                "type") == "journal":
                                            url = openurl_url.text
                                        elif openurl_url.get(
                                                "type") == "source":
                                            url = openurl_url.text
                                if url:
                                    resource = Resource()
                                    resource["rec_type"] = "ResourceRemote"
                                    resource["rec_state"] = "published"
                                    resource["relation_type"] = "eResource"
                                    resource[
                                        "version_type"] = "publishedVersion"
                                    resource["access_rights"] = "closedAccess"
                                    resource["format_mimetype"] = "text/html"
                                    resource["url"] = url
                                    if service_name:
                                        resource["service_name"] = service_name
                                    if institution_name:
                                        resource[
                                            "institution_name"] = institution_name
                                    if period_begin:
                                        resource["period_begin"] = period_begin
                                    if period_end:
                                        resource["period_end"] = period_end
                                    document.add_item_to_key(
                                        resource, "resources")
                    documents.append(document)
                    if only_first_record:
                        break
    #logging.debug(jsonbson.dumps_json(documents))
    return documents
Exemple #9
0
def csv_dict_reader_to_metasjon(csv_row, input_format, source, rec_id_prefix):
    document = Document()

    if source:
        document["rec_source"] = source

    if input_format == constants.FORMAT_CSV_SITPOL:
        #logging.debug("csv_dict_reader_to_metasjon type(csv_row): {}".format(type(csv_row)))
        #print csv_row
        document["title"] = csv_row["title"]
        classifications_sitpol = [x.strip() for x in csv_row["classifications_sitpol"].split(";") if x.strip()]
        if classifications_sitpol:
            document["classifications_sitpol"] = classifications_sitpol
        classifications_ddc = [x.strip() for x in csv_row["classifications_ddc"].split(";") if x.strip()]
        if classifications_ddc:
            document["classifications_ddc"] = classifications_ddc
        formatted_names = [x.strip() for x in csv_row["creators@role=pbl"].split(";") if x.strip()]
        if formatted_names:
            #logging.debug("formatted_names: {}".format(formatted_names))
            creators = []
            for formatted_name in formatted_names:
                if formatted_name:
                    creator = creator_service.formatted_name_to_creator(formatted_name, None, "pbl")
                    if creator:
                        creators.append(creator)
            if creators:
                document["creators"] = creators
        document["date_last_accessed"] = datetime.now().isoformat()
        document["descriptions"] = [{"language":"fr", "value":csv_row["descriptions@lang=fr"]}]
        keywords_fr = [x.strip() for x in csv_row["keywords@lang=fr"].split(";") if x.strip()]
        keywords_en = [x.strip() for x in csv_row["keywords@lang=en"].split(";") if x.strip()]
        keywords = {}
        if keywords_fr:
            keywords["fr"] = keywords_fr
        if keywords_en:
            keywords["en"] = keywords_en
        if keywords:
            document["keywords"] = keywords
        document["languages"] = [x.strip() for x in csv_row["languages"].split(";") if x.strip()]
        note = csv_row["notes@lang=fr"]
        if note:
            document["notes"] = note
        document["publication_countries"] = [x.strip() for x in csv_row["publication_countries"].split(";") if x.strip()]
        if "rec_created_user" in csv_row:
            document["rec_created_user"] = csv_row["rec_created_user"]
        document["rec_type_cerimes"] = csv_row["rec_type_cerimes"]
        specific_agents = [x.strip() for x in csv_row["specific_agents"].split(";") if x.strip()]
        if specific_agents:
            document["specific_agents"] = specific_agents
        document["specific_actor_type"] = csv_row["specific_actor_type"]
        document["target_audiences_cerimes"] = csv_row["target_audiences_cerimes"]
        document["url"] = csv_row["url"]
        document["rec_type"] = constants.DOC_TYPE_WEBENTITY
        document["webentity_type"] = csv_row["webentity_type"]
    elif input_format == constants.FORMAT_CSV_METAJSON:
        document["rec_type"] = "DatasetQuali"
        creators = []
        if "Laboratoire d'inventaire" in csv_row:
            creators.append(creator_service.formatted_name_to_creator(csv_row["Laboratoire d'inventaire"], constants.REC_CLASS_ORGUNIT, "dpt"))
        document["title"] = csv_row["Titre de l'enquete"]
        if "Sujet(s) de l'enquete" in csv_row:
            document["keywords"] = [x.strip() for x in csv_row["Sujet(s) de l'enquete"].split("\n") if x.strip()]

        if "Nom Auteur 1" in csv_row:
            name_familly = csv_row["Nom Auteur 1"]
            name_given = affiliation = ""
            if "Prenom Auteur 1" in csv_row:
                name_given = csv_row["Prenom Auteur 1"]
            if "Affiliation Auteur 1" in csv_row:
                affiliation = csv_row["Affiliation Auteur 1"]


        document["creators"] = creators
    else:
        logging.error("Unknown input_format: {}".format(input_format))

    logging.info(jsonbson.dumps_json(document, True))
    return document
Exemple #10
0
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix,
                                   only_first_record):
    document = None
    ris_type = None
    rec_type = None
    is_part_of_rec_type = None
    previous_key = None
    previous_value = None
    for line in txt_lines:
        if line:
            line = line.rstrip('\r\n')
            #logging.debug("line: {}".format(line))

            # multi line management
            if previous_key:
                key = previous_key
                value = previous_value + line
                previous_key = None
                previous_value = None
            else:
                key = line[:2].strip()
                value = line[6:].strip()
            if value.endswith("/") and key not in ["Y1", "PY"]:
                #logging.debug("multi line")
                previous_key = key
                previous_value = value.rstrip('/')
                continue

            if key is None or len(key) == 0:
                # empty line -> continue
                #logging.debug("empty line")
                continue
            elif key == RIS_KEY_BEGIN:
                # record begin with document type -> create document
                # init
                document = Document()
                is_part_of_rec_type = None

                if source:
                    document["rec_source"] = source
                ris_type = value
                rec_type = ris_document_type_to_metajson_document_type[
                    ris_type]
                document["rec_type"] = rec_type
                if ris_type in ris_document_type_to_metajson_document_is_part_of_type:
                    is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[
                        ris_type]
                    is_part_of = Document()
                    is_part_of["rec_type"] = is_part_of_rec_type
                    document["is_part_ofs"] = [is_part_of]
            elif key == RIS_KEY_END:
                # record end -> return the result
                # verify the is_part_ofs[0]["title"]
                if "is_part_ofs" in document and "title" not in document[
                        "is_part_ofs"][0] and "title_abbreviateds" in document[
                            "is_part_ofs"][0]:
                    document["is_part_ofs"][0]["title"] = document[
                        "is_part_ofs"][0]["title_abbreviateds"][0]["title"]
                    del document["is_part_ofs"][0]["title_abbreviateds"]

                logging.info("# RIS type: {}".format(ris_type))
                metajson_service.pretty_print_document(document)
                yield document
            else:
                # process key value
                #logging.debug("key: {}; value: {}".format(key, value))
                if key == "ID":
                    document["rec_id"] = value
                elif key in [
                        "T1", "TI", "CT"
                ] or (key == "BT"
                      and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]):
                    # Title Primary -> title
                    document["title"] = value
                elif key in [
                        "JF", "JO"
                ] or (key == "BT"
                      and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]):
                    # Title Secondary -> is_part_of["title"]
                    document.add_is_part_of_title(value)
                elif key in ["JA", "J1", "J2", "T2"]:
                    # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"]
                    document.add_is_part_of_title_abbreviated(value)
                elif key == "T3":
                    # Title Series
                    document.add_series_title(value)
                elif key in ["A1", "AU"]:
                    document.add_creator(
                        creator_service.formatted_name_to_creator(
                            value, None, "aut"))
                elif key in ["A2", "ED"]:
                    if is_part_of_rec_type:
                        document.add_is_part_of_creator(
                            creator_service.formatted_name_to_creator(
                                value, None, "edt"))
                    else:
                        document.add_creator(
                            creator_service.formatted_name_to_creator(
                                value, None, "edt"))
                elif key == "A3":
                    document.add_series_creator(
                        creator_service.formatted_name_to_creator(
                            value, None, "aut"))
                elif key == "A4":
                    document.add_creator(
                        creator_service.formatted_name_to_creator(
                            value, None, "ctb"))
                elif key in ["PY", "Y1", "DA"]:
                    index_slash = value.find("/")
                    if index_slash != -1:
                        # YYYY/MM/DD/other info (like season)
                        # todo
                        document["date_issued"] = value.strip("/")
                    else:
                        document["date_issued"] = value
                elif key == "SP":
                    document["part_page_begin"] = value
                elif key == "EP":
                    document["part_page_end"] = value
                elif key == "VL":
                    document["part_volume"] = value
                elif key in ["IS", "CP"]:
                    document["part_issue"] = value
                elif key in ["AB", "N2"]:
                    document["descriptions"] = [{
                        "language": "und",
                        "value": value
                    }]
                elif key == "N1":
                    document["notes"] = [{"language": "und", "value": value}]
                elif key == "PB":
                    document.add_item_to_key(value, "publishers")
                elif key == "CY":
                    document.add_item_to_key(value, "publication_places")
                elif key == "RP":
                    document["publication_status"] = value
                elif key == "ET":
                    document["edition"] = value
                elif key == "UR":
                    resource = Resource()
                    resource["url"] = value
                    document.add_item_to_key(resource, "resources")
                elif key == "AN":
                    # Accession Number
                    identifier = metajson_service.create_identifier(
                        "accessionnumber", value)
                    document.add_identifier(identifier)
                elif key == "CN":
                    # Call Number
                    identifier = metajson_service.create_identifier(
                        "callnumber", value)
                    document.add_identifier(identifier)
                elif key == "DO":
                    # DOI
                    identifier = metajson_service.create_identifier(
                        "doi", value)
                    document.add_identifier(identifier)
                elif key == "SN":
                    # ISBN or ISSN ?
                    id_type = None
                    if rec_type in [
                            constants.DOC_TYPE_JOURNALARTICLE,
                            constants.DOC_TYPE_MAGAZINEARTICLE,
                            constants.DOC_TYPE_NEWSPAPERARTICLE,
                            constants.DOC_TYPE_JOURNAL
                    ]:
                        id_type = "issn"
                    else:
                        id_type = "isbn"
                    identifier = metajson_service.create_identifier(
                        id_type, value)
                    if is_part_of_rec_type is None:
                        document.add_identifier(identifier)
                    else:
                        document["is_part_ofs"][0].add_identifier(identifier)
                elif key == "CA":
                    document["caption"] = value
                elif key == "DB":
                    # Name of Database -> rec_source ?
                    document["rec_source"] = value
                elif key == "DP":
                    # NDatabase Provider -> rec_source ?
                    document["rec_source"] = value
                elif key == "KW":
                    if "keywords" not in document:
                        document["keywords"] = {"und": []}
                    document["keywords"]["und"].append(value)
                else:
                    logging.debug("Not managed key: {} with value: {}".format(
                        key, value))
def openurl_xmletree_to_metajson_list(openurl_response, source, rec_id_prefix, only_first_record):
    documents = []
    if openurl_response is not None:
        #logging.debug(type(openurl_response))
        #logging.debug(openurl_response)
        # results
        openurl_results = openurl_response.find(xmletree.prefixtag("ssopenurl", "results"))
        if openurl_results is not None:
            # result
            openurl_result_list = openurl_results.findall(xmletree.prefixtag("ssopenurl", "result"))
            if openurl_result_list:
                for openurl_result in openurl_result_list:
                    document = Document()
                    if source:
                        document["source"] = source
                    # citation
                    openurl_citation = openurl_result.find(xmletree.prefixtag("ssopenurl", "citation"))
                    if openurl_citation is not None:
                        # issn
                        openurl_issn = openurl_citation.find(xmletree.prefixtag("ssopenurl", "issn"))
                        if openurl_issn is not None:
                            identifier_issn = Identifier()
                            identifier_issn["id_type"] = "issn"
                            identifier_issn["value"] = openurl_issn.text
                            document.add_item_to_key(identifier_issn, "identifiers")
                        # eissn
                        openurl_eissn = openurl_citation.find(xmletree.prefixtag("ssopenurl", "eissn"))
                        if openurl_eissn is not None:
                            identifier_eissn = Identifier()
                            identifier_eissn["id_type"] = "eissn"
                            identifier_eissn["value"] = openurl_eissn.text
                            document.add_item_to_key(identifier_eissn, "identifiers")
                    # linkGroups
                    openurl_linkgroups = openurl_result.find(xmletree.prefixtag("ssopenurl", "linkGroups"))
                    if openurl_linkgroups is not None:
                        # linkGroup
                        openurl_linkgroup_list = openurl_linkgroups.findall(xmletree.prefixtag("ssopenurl", "linkGroup"))
                        if openurl_linkgroup_list is not None:
                            for openurl_linkgroup in openurl_linkgroup_list:
                                service_name = None
                                institution_name = None
                                period_begin = None
                                period_end = None
                                url = None
                                # holdingData
                                openurl_holdingdata = openurl_linkgroup.find(xmletree.prefixtag("ssopenurl", "holdingData"))
                                if openurl_holdingdata is not None:
                                    # institution_name
                                    openurl_providername = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "providerName"))
                                    if openurl_providername is not None:
                                        institution_name = openurl_providername.text
                                    # service_name
                                    openurl_databasename = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "databaseName"))
                                    if openurl_databasename is not None:
                                        service_name = openurl_databasename.text
                                    # normalizedData
                                    openurl_normalizeddata = openurl_holdingdata.find(xmletree.prefixtag("ssopenurl", "normalizedData"))
                                    if openurl_normalizeddata is not None:
                                        # startDate
                                        openurl_startdate = openurl_normalizeddata.find(xmletree.prefixtag("ssopenurl", "startDate"))
                                        if openurl_startdate is not None:
                                            period_begin = openurl_startdate.text
                                        # endDate
                                        openurl_enddate = openurl_normalizeddata.find(xmletree.prefixtag("ssopenurl", "endDate"))
                                        if openurl_enddate is not None:
                                            period_end = openurl_enddate.text
                                # url
                                openurl_url_list = openurl_linkgroup.findall(xmletree.prefixtag("ssopenurl", "url"))
                                if openurl_url_list is not None:
                                    for openurl_url in openurl_url_list:
                                        if openurl_url.get("type") == "journal":
                                            url = openurl_url.text
                                        elif openurl_url.get("type") == "source":
                                            url = openurl_url.text
                                if url:
                                    resource = Resource()
                                    resource["rec_type"] = "ResourceRemote"
                                    resource["rec_state"] = "published"
                                    resource["relation_type"] = "eResource"
                                    resource["version_type"] = "publishedVersion"
                                    resource["access_rights"] = "closedAccess"
                                    resource["format_mimetype"] = "text/html"
                                    resource["url"] = url
                                    if service_name:
                                        resource["service_name"] = service_name
                                    if institution_name:
                                        resource["institution_name"] = institution_name
                                    if period_begin:
                                        resource["period_begin"] = period_begin
                                    if period_end:
                                        resource["period_end"] = period_end
                                    document.add_item_to_key(resource, "resources")
                    documents.append(document)
                    if only_first_record:
                        break
    #logging.debug(jsonbson.dumps_json(documents))
    return documents
Exemple #12
0
def endnotexml_record_to_metajson(record, source, rec_id_prefix):
    document = Document()

    # TODO
    # translated_creators: /contributors/translated-authors/author/style
    # auth_address: /auth-address/style
    # label: /label/style
    # custom1

    # Extract endnote properties

    rec_id = record.find("rec-number").text
    endnote_type = record.find("ref-type").text
    rec_type = endnote_record_type_to_metajson_document_type[endnote_type]

    primary_creators = extract_creators(None, "aut", record,
                                        "./contributors/authors/author/style")
    secondary_creators = extract_creators(
        None, "pbd", record, "./contributors/secondary-authors/author/style")
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        tertiary_creators = extract_creators(
            None, "pbd", record,
            "./contributors/tertiary-authors/author/style")
    elif endnote_type == TYPE_THESIS:
        tertiary_creators = extract_creators(
            None, "ths", record,
            "./contributors/tertiary-authors/author/style")
    elif endnote_type == TYPE_FILM_OR_BROADCAST:
        tertiary_creators = extract_creators(
            None, "pro", record,
            "./contributors/tertiary-authors/author/style")
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        subsidiary_creators = extract_creators(
            None, "trl", record,
            "./contributors/subsidiary-authors/author/style")
    elif endnote_type == TYPE_FILM_OR_BROADCAST:
        subsidiary_creators = extract_creators(
            None, "act", record,
            "./contributors/subsidiary-authors/author/style")
    translated_creators = extract_creators(
        None, "trl", record, "./contributors/translated-authors/author/style")
    auth_address = extract_text(record, "./auth-address/style")

    title = extract_text(record, "./titles/title/style")
    title_secondary = extract_text(record, "./titles/secondary-title/style")
    title_tertiary = extract_text(record, "./titles/tertiary-title/style")
    title_alternative = extract_text(record, "./titles/alt-title/style")
    title_abbreviated = extract_text(record, "./titles/short-title/style")
    title_translated = extract_text(record, "./titles/translated-title/style")

    pages = extract_text(record, "./pages/style")
    part_volume = extract_text(record, "./volume/style")
    part_number = extract_text(record, "./number/style")
    extent_volumes = extract_text(record, "./num-vols/style")
    edition = extract_text(record, "./edition/style")
    part_section = extract_text(record, "./section/style")
    reprint_edition = extract_text(record, "./reprint-edition/style")
    keywords = extract_text(record, "./keywords/keyword/style")
    date_year = extract_text(record, "./dates/year/style")
    date_pub = extract_text(record, "./dates/pub-dates/date/style")
    publication_places_formatted = extract_text(record, "./pub-location/style")
    publishers_formatted = extract_text(record, "./publisher/style")
    orig_pub = extract_text(record, "./orig-pub/style")
    isbn_or_issn = extract_text(record, "./isbn/style")
    accessionnumber = extract_text(record, "./accession-num/style")
    callnumber = extract_text(record, "./call-num/style")
    if endnote_type == TYPE_WEB_PAGE:
        abstract = extract_text(record, "./pages/style")
    else:
        abstract = extract_text(record, "./abstract/style")
    label = extract_text(record, "./label/style")
    caption = extract_text(record, "./caption/style")
    note = extract_text(record, "./notes/style")
    reviewed_item = extract_text(record, "./reviewed-item/style")
    rec_type_description = extract_text(record, "./work-type/style")
    url = extract_text(record, "./urls/related-urls/url/style")
    custom1 = extract_text(record, "./custom1/style")
    custom2 = extract_text(record, "./custom2/style")
    custom3 = extract_text(record, "./custom3/style")
    custom4 = extract_text(record, "./custom4/style")
    custom5 = extract_text(record, "./custom5/style")
    custom6 = extract_text(record, "./custom6/style")
    custom7 = extract_text(record, "./custom7/style")
    doi = extract_text(record, "./electronic-resource-num/style")
    remote_database_name = extract_text(record, "./remote-database-name/style")
    remote_database_provider = extract_text(
        record, "./remote-database-provider/style")
    research_notes = extract_text(record, "./research-notes/style")
    language = extract_text(record, "./language/style")
    access_date = extract_text(record, "./access-date/style")

    # rec_id, rec_source
    document["rec_id"] = rec_id
    if source:
        document["rec_source"] = source

    # publishers_formatted, publication_places_formatted
    publishers = None
    publication_places = None
    if publishers_formatted:
        publishers = publishers_formatted.split("\r")
    if publication_places_formatted:
        publication_places = publication_places_formatted.split("\r")

    # type, is_part_of.type, is_part_of.is_part_of.type
    try:
        is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[
            endnote_type]
    except:
        is_part_of_type = None

    is_part_of_is_part_of_type = None
    if title_secondary is not None:
        if endnote_type == TYPE_FIGURE:
            # how to determine the is_part_of type ?
            # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart
            if part_volume is not None or part_number is not None:
                is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE
                is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL
            else:
                if title_translated is not None:
                    is_part_of_type = constants.DOC_TYPE_BOOKPART
                    is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK
                else:
                    is_part_of_type = constants.DOC_TYPE_BOOK
        elif endnote_type == TYPE_FILM_OR_BROADCAST:
            rec_type = constants.DOC_TYPE_VIDEOPART
            is_part_of_type = constants.DOC_TYPE_VIDEORECORDING

    document["rec_type"] = rec_type
    document.set_key_if_not_none("rec_type_description", rec_type_description)

    # issn or isbn ?
    if is_part_of_type in [
            constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER,
            constants.DOC_TYPE_MAGAZINE
    ]:
        isbn_or_issn_type = "issn"
    else:
        isbn_or_issn_type = "isbn"

    # is_part_of, is_part_of.is_part_of
    if is_part_of_type is not None and title_secondary:
        is_part_of = Document()
        is_part_of.set_key_if_not_none("rec_type", is_part_of_type)
        is_part_of.set_key_if_not_none("title", title_secondary)

        if is_part_of_is_part_of_type is not None:
            # is_part_of in case of is_part_of.is_part_of
            # creators with role aut
            is_part_of.add_creators(
                creator_service.change_contibutors_role(
                    secondary_creators, "aut"))

            # is_part_of.is_part_of
            is_part_of_is_part_of = Document()
            is_part_of_is_part_of.set_key_if_not_none(
                "rec_type", is_part_of_is_part_of_type)
            is_part_of_is_part_of.set_key_if_not_none("title",
                                                      title_translated)
            # creators with role pbd
            is_part_of_is_part_of.add_creators(
                creator_service.change_contibutors_role(
                    translated_creators, "pbd"))
            #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year)
            is_part_of_is_part_of.set_key_if_not_none("publishers", publishers)
            is_part_of_is_part_of.set_key_if_not_none("publication_places",
                                                      publication_places)
            if isbn_or_issn:
                is_part_of["identifiers"] = [
                    metajson_service.create_identifier(isbn_or_issn_type,
                                                       isbn_or_issn)
                ]

            is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs")

        else:
            # is_part_of in case of no is_part_of.is_part_of
            # creators with role edt
            is_part_of.add_creators(secondary_creators)
            #is_part_of.set_key_if_not_none("date_issued",date_year)
            is_part_of.set_key_if_not_none("publishers", publishers)
            is_part_of.set_key_if_not_none("publication_places",
                                           publication_places)
            if isbn_or_issn:
                is_part_of["identifiers"] = [
                    metajson_service.create_identifier(isbn_or_issn_type,
                                                       isbn_or_issn)
                ]

        if "title" in is_part_of and is_part_of["title"]:
            document.add_items_to_key([is_part_of], "is_part_ofs")

    else:
        if isbn_or_issn:
            document["identifiers"] = [
                metajson_service.create_identifier(isbn_or_issn_type,
                                                   isbn_or_issn)
            ]
        if publishers:
            if endnote_type == TYPE_THESIS:
                document.add_creators([
                    creator_service.formatted_name_to_creator(
                        publishers[0], "orgunit", "dgg")
                ])
            elif endnote_type == TYPE_FILM_OR_BROADCAST:
                document.add_creators([
                    creator_service.formatted_name_to_creator(
                        publishers[0], "orgunit", "dst")
                ])
            else:
                document.set_key_if_not_none("publishers", publishers)
        document.set_key_if_not_none("publication_places", publication_places)

    # seriess[]
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        series = Document()
        series["rec_type"] = constants.DOC_TYPE_SERIES
        if endnote_type == TYPE_BOOK and title_secondary:
            series.set_key_if_not_none("title", title_secondary)
            series.add_creators(secondary_creators)
            series.set_key_if_not_none("part_volume", part_number)
        if endnote_type == TYPE_BOOK_SECTION and title_tertiary:
            series.set_key_if_not_none("title", title_tertiary)
            series.add_creators(tertiary_creators)
            series.set_key_if_not_none("part_volume", part_number)
        if "title" in series and len(series) > 2:
            document.add_items_to_key([series], "seriess")

    # originals[]
    if (reprint_edition or title_translated or orig_pub) and endnote_type in [
            TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE,
            TYPE_FILM_OR_BROADCAST
    ]:
        original_title = None
        original_is_part_of = None
        if reprint_edition:
            original_title = reprint_edition
        if title_translated and is_part_of_is_part_of_type is None:
            original_title = title_translated
        if orig_pub:
            if is_part_of_type is not None:
                original_is_part_of = Document()
                original_is_part_of["rec_type"] = is_part_of_type
                original_is_part_of["title"] = orig_pub
            else:
                original_title = orig_pub
        if original_title:
            original = Document()
            original["rec_type"] = rec_type
            original.set_key_if_not_none("title", original_title)
            original.add_item_to_key(original_is_part_of, "is_part_ofs")
            document.add_item_to_key(original, "originals")

    # is_review_ofs[]
    if reviewed_item and endnote_type in [
            TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE
    ]:
        is_review_ofs = Document()
        is_review_ofs.set_key_if_not_none("title", reviewed_item)
        is_review_ofs.set_key_if_not_none("rec_type", "Book")
        document.add_items_to_key([is_review_ofs], "is_review_ofs")

    # descriptions[0].value
    if abstract:
        document["descriptions"] = [{"value": abstract, "language": "und"}]

    # archive
    if endnote_type == TYPE_FIGURE and remote_database_provider:
        archive = Document()
        archive["title"] = remote_database_provider
        document.add_items_to_key([archive], "archive")

    # caption
    document.set_key_if_not_none("caption", caption)

    # creators[]
    document.add_creators(primary_creators)
    if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]:
        document.add_creators(tertiary_creators)
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]:
        document.add_creators(subsidiary_creators)
    if custom4:
        document.add_creators(
            endnote_authors_to_creators(custom4, "person", "rev"))
    if endnote_type == TYPE_FIGURE and remote_database_name:
        document.add_creators(
            endnote_authors_to_creators(remote_database_name, None, "cph"))

    # edition
    if endnote_type in [
            TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE
    ] and edition:
        document["edition"] = edition

    # extent_pages, extent_volumes
    if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages:
        document["extent_pages"] = pages.replace("p.", "").strip()
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes:
        document["extent_volumes"] = extent_volumes

    # date_issued, date_issued_first
    if date_year:
        date_issued = ""
        date_issued_first = ""
        orig_index_begin = date_year.find("[")
        orig_index_end = date_year.find("]")
        if orig_index_begin != -1 and orig_index_end != -1:
            date_issued_first = date_year[orig_index_begin + 1:orig_index_end]
            date_issued = date_year.replace("[" + date_issued_first + "]",
                                            "").strip()
        else:
            date_issued = date_year.strip()

        if "is_part_ofs" in document:
            if document["is_part_ofs"][0][
                    "rec_type"] == constants.DOC_TYPE_BOOK:
                document["is_part_ofs"][0].set_key_if_not_none(
                    "date_issued", date_issued)
                document["is_part_ofs"][0].set_key_if_not_none(
                    "date_issued_first", date_issued_first)
            elif "is_part_ofs" in document["is_part_ofs"][0] and document[
                    "is_part_ofs"][0]["is_part_ofs"][0][
                        "rec_type"] == constants.DOC_TYPE_BOOK:
                document["is_part_ofs"][0]["is_part_ofs"][
                    0].set_key_if_not_none("date_issued", date_issued)
                document["is_part_ofs"][0]["is_part_ofs"][
                    0].set_key_if_not_none("date_issued_first",
                                           date_issued_first)
            else:
                document.set_key_if_not_none("date_issued_first",
                                             date_issued_first)
                if rec_type in unpublished_types:
                    document.set_key_if_not_none("date_created", date_issued)
                else:
                    document.set_key_if_not_none("date_issued", date_issued)
        else:
            document.set_key_if_not_none("date_issued_first",
                                         date_issued_first)
            if rec_type in unpublished_types:
                document.set_key_if_not_none("date_created", date_issued)
            else:
                document.set_key_if_not_none("date_issued", date_issued)

    # identifiers[]
    identifiers = []
    if accessionnumber:
        identifiers.append(
            metajson_service.create_identifier("accessionnumber",
                                               accessionnumber))
    if callnumber:
        identifiers.append(
            metajson_service.create_identifier("callnumber", callnumber))
    if doi:
        identifiers.append(metajson_service.create_identifier("doi", doi))
    if identifiers:
        document["identifiers"] = identifiers

    # language
    if language:
        rfc5646 = language_service.convert_unknown_format_to_rfc5646(language)
        if rfc5646:
            document["languages"] = [rfc5646]

    # note
    if endnote_import_note and note:
        document.set_key_with_value_type_in_list("notes", note, "general")
    if endnote_import_research_note and research_notes:
        document.set_key_with_value_type_in_list("notes", research_notes,
                                                 "user")

    # part_page_begin & part_page_end
    if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE
                        ] and pages:
        hyphen_index = pages.find("-")
        if hyphen_index == -1:
            document["part_page_begin"] = pages.replace("p.", "").strip()
        else:
            document["part_page_begin"] = pages[:hyphen_index].replace(
                "p.", "").strip()
            document["part_page_end"] = pages[hyphen_index + 1:].replace(
                "p.", "").strip()

    if endnote_type in [TYPE_JOURNAL_ARTICLE]:
        document.set_key_if_not_none("part_issue", part_number)
    elif endnote_type in [TYPE_FIGURE]:
        document.set_key_if_not_none("part_number", part_number)
    document.set_key_if_not_none("part_section", part_section)
    document.set_key_if_not_none("part_volume", part_volume)

    # resources[0]
    if url is not None:
        resource = Resource()
        resource["rec_type"] = "ResourceRemote"
        resource.set_key_if_not_none("url", url)
        if endnote_type == TYPE_WEB_PAGE:
            resource.set_key_if_not_none("date_last_accessed", part_number)
        else:
            resource.set_key_if_not_none("date_last_accessed", access_date)
        document["resources"] = [resource]

    # subjects[]
    if endnote_import_keywords and keywords:
        for keyword in keywords.split():
            document.set_key_with_value_type_in_list("subjects", keyword,
                                                     "topic")

    # title, title_alternative, title_abbreviated, title_translated
    document["title"] = title
    if title_alternative:
        document["title_alternatives"] = [{"title": title_alternative}]
    if title_abbreviated:
        document["title_abbreviateds"] = [{"title": title_abbreviated}]

    #logging.debug("# endnote_type: {}".format(endnote_type))
    metajson_service.pretty_print_document(document)
    return document
def endnotexml_record_to_metajson(record, source, rec_id_prefix):
    document = Document()

    # TODO
    # translated_creators: /contributors/translated-authors/author/style
    # auth_address: /auth-address/style
    # label: /label/style
    # custom1

    # Extract endnote properties

    rec_id = record.find("rec-number").text
    endnote_type = record.find("ref-type").text
    rec_type = endnote_record_type_to_metajson_document_type[endnote_type]

    primary_creators = extract_creators(None, "aut", record, "./contributors/authors/author/style")
    secondary_creators = extract_creators(None, "pbd", record, "./contributors/secondary-authors/author/style")
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        tertiary_creators = extract_creators(None, "pbd", record, "./contributors/tertiary-authors/author/style")
    elif endnote_type == TYPE_THESIS:
        tertiary_creators = extract_creators(None, "ths", record, "./contributors/tertiary-authors/author/style")
    elif endnote_type == TYPE_FILM_OR_BROADCAST:
        tertiary_creators = extract_creators(None, "pro", record, "./contributors/tertiary-authors/author/style")
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        subsidiary_creators = extract_creators(None, "trl", record, "./contributors/subsidiary-authors/author/style")
    elif endnote_type == TYPE_FILM_OR_BROADCAST:
        subsidiary_creators = extract_creators(None, "act", record, "./contributors/subsidiary-authors/author/style")
    translated_creators = extract_creators(None, "trl", record, "./contributors/translated-authors/author/style")
    auth_address = extract_text(record, "./auth-address/style")

    title = extract_text(record, "./titles/title/style")
    title_secondary = extract_text(record, "./titles/secondary-title/style")
    title_tertiary = extract_text(record, "./titles/tertiary-title/style")
    title_alternative = extract_text(record, "./titles/alt-title/style")
    title_abbreviated = extract_text(record, "./titles/short-title/style")
    title_translated = extract_text(record, "./titles/translated-title/style")

    pages = extract_text(record, "./pages/style")
    part_volume = extract_text(record, "./volume/style")
    part_number = extract_text(record, "./number/style")
    extent_volumes = extract_text(record, "./num-vols/style")
    edition = extract_text(record, "./edition/style")
    part_section = extract_text(record, "./section/style")
    reprint_edition = extract_text(record, "./reprint-edition/style")
    keywords = extract_text(record, "./keywords/keyword/style")
    date_year = extract_text(record, "./dates/year/style")
    date_pub = extract_text(record, "./dates/pub-dates/date/style")
    publication_places_formatted = extract_text(record, "./pub-location/style")
    publishers_formatted = extract_text(record, "./publisher/style")
    orig_pub = extract_text(record, "./orig-pub/style")
    isbn_or_issn = extract_text(record, "./isbn/style")
    accessionnumber = extract_text(record, "./accession-num/style")
    callnumber = extract_text(record, "./call-num/style")
    if endnote_type == TYPE_WEB_PAGE:
        abstract = extract_text(record, "./pages/style")
    else:
        abstract = extract_text(record, "./abstract/style")
    label = extract_text(record, "./label/style")
    caption = extract_text(record, "./caption/style")
    note = extract_text(record, "./notes/style")
    reviewed_item = extract_text(record, "./reviewed-item/style")
    rec_type_description = extract_text(record, "./work-type/style")
    url = extract_text(record, "./urls/related-urls/url/style")
    custom1 = extract_text(record, "./custom1/style")
    custom2 = extract_text(record, "./custom2/style")
    custom3 = extract_text(record, "./custom3/style")
    custom4 = extract_text(record, "./custom4/style")
    custom5 = extract_text(record, "./custom5/style")
    custom6 = extract_text(record, "./custom6/style")
    custom7 = extract_text(record, "./custom7/style")
    doi = extract_text(record, "./electronic-resource-num/style")
    remote_database_name = extract_text(record, "./remote-database-name/style")
    remote_database_provider = extract_text(record, "./remote-database-provider/style")
    research_notes = extract_text(record, "./research-notes/style")
    language = extract_text(record, "./language/style")
    access_date = extract_text(record, "./access-date/style")

    # rec_id, rec_source
    document["rec_id"] = rec_id
    if source:
        document["rec_source"] = source

    # publishers_formatted, publication_places_formatted
    publishers = None
    publication_places = None
    if publishers_formatted:
        publishers = publishers_formatted.split("\r")
    if publication_places_formatted:
        publication_places = publication_places_formatted.split("\r")

    # type, is_part_of.type, is_part_of.is_part_of.type
    try:
        is_part_of_type = endnote_record_type_to_metajson_document_is_part_of_type[endnote_type]
    except:
        is_part_of_type = None

    is_part_of_is_part_of_type = None
    if title_secondary is not None:
        if endnote_type == TYPE_FIGURE:
            # how to determine the is_part_of type ?
            # if there is a volume or an issue number, it's a JournalArticle, else it's a Book or BookPart
            if part_volume is not None or part_number is not None:
                is_part_of_type = constants.DOC_TYPE_JOURNALARTICLE
                is_part_of_is_part_of_type = constants.DOC_TYPE_JOURNAL
            else:
                if title_translated is not None:
                    is_part_of_type = constants.DOC_TYPE_BOOKPART
                    is_part_of_is_part_of_type = constants.DOC_TYPE_BOOK
                else:
                    is_part_of_type = constants.DOC_TYPE_BOOK
        elif endnote_type == TYPE_FILM_OR_BROADCAST:
            rec_type = constants.DOC_TYPE_VIDEOPART
            is_part_of_type = constants.DOC_TYPE_VIDEORECORDING

    document["rec_type"] = rec_type
    document.set_key_if_not_none("rec_type_description", rec_type_description)

    # issn or isbn ?
    if is_part_of_type in [constants.DOC_TYPE_JOURNAL, constants.DOC_TYPE_NEWSPAPER, constants.DOC_TYPE_MAGAZINE]:
        isbn_or_issn_type = "issn"
    else:
        isbn_or_issn_type = "isbn"

    # is_part_of, is_part_of.is_part_of
    if is_part_of_type is not None and title_secondary:
        is_part_of = Document()
        is_part_of.set_key_if_not_none("rec_type", is_part_of_type)
        is_part_of.set_key_if_not_none("title", title_secondary)

        if is_part_of_is_part_of_type is not None:
            # is_part_of in case of is_part_of.is_part_of
            # creators with role aut
            is_part_of.add_creators(creator_service.change_contibutors_role(secondary_creators, "aut"))

            # is_part_of.is_part_of
            is_part_of_is_part_of = Document()
            is_part_of_is_part_of.set_key_if_not_none("rec_type", is_part_of_is_part_of_type)
            is_part_of_is_part_of.set_key_if_not_none("title", title_translated)
            # creators with role pbd
            is_part_of_is_part_of.add_creators(creator_service.change_contibutors_role(translated_creators, "pbd"))
            #is_part_of_is_part_of.set_key_if_not_none("date_issued",date_year)
            is_part_of_is_part_of.set_key_if_not_none("publishers", publishers)
            is_part_of_is_part_of.set_key_if_not_none("publication_places", publication_places)
            if isbn_or_issn:
                is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)]

            is_part_of.add_items_to_key([is_part_of_is_part_of], "is_part_ofs")

        else:
            # is_part_of in case of no is_part_of.is_part_of
            # creators with role edt
            is_part_of.add_creators(secondary_creators)
            #is_part_of.set_key_if_not_none("date_issued",date_year)
            is_part_of.set_key_if_not_none("publishers", publishers)
            is_part_of.set_key_if_not_none("publication_places", publication_places)
            if isbn_or_issn:
                is_part_of["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)]

        if "title" in is_part_of and is_part_of["title"]:
            document.add_items_to_key([is_part_of], "is_part_ofs")

    else:
        if isbn_or_issn:
            document["identifiers"] = [metajson_service.create_identifier(isbn_or_issn_type, isbn_or_issn)]
        if publishers:
            if endnote_type == TYPE_THESIS:
                document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dgg")])
            elif endnote_type == TYPE_FILM_OR_BROADCAST:
                document.add_creators([creator_service.formatted_name_to_creator(publishers[0], "orgunit", "dst")])
            else:
                document.set_key_if_not_none("publishers", publishers)
        document.set_key_if_not_none("publication_places", publication_places)

    # seriess[]
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION]:
        series = Document()
        series["rec_type"] = constants.DOC_TYPE_SERIES
        if endnote_type == TYPE_BOOK and title_secondary:
            series.set_key_if_not_none("title", title_secondary)
            series.add_creators(secondary_creators)
            series.set_key_if_not_none("part_volume", part_number)
        if endnote_type == TYPE_BOOK_SECTION and title_tertiary:
            series.set_key_if_not_none("title", title_tertiary)
            series.add_creators(tertiary_creators)
            series.set_key_if_not_none("part_volume", part_number)
        if "title" in series and len(series) > 2:
            document.add_items_to_key([series], "seriess")

    # originals[]
    if (reprint_edition or title_translated or orig_pub) and endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE, TYPE_FILM_OR_BROADCAST]:
        original_title = None
        original_is_part_of = None        
        if reprint_edition:
            original_title = reprint_edition
        if title_translated and is_part_of_is_part_of_type is None:
            original_title = title_translated
        if orig_pub:
            if is_part_of_type is not None:
                original_is_part_of = Document()
                original_is_part_of["rec_type"] = is_part_of_type
                original_is_part_of["title"] = orig_pub
            else:
                original_title = orig_pub
        if original_title:
            original = Document()
            original["rec_type"] = rec_type
            original.set_key_if_not_none("title", original_title)
            original.add_item_to_key(original_is_part_of, "is_part_ofs")
            document.add_item_to_key(original, "originals")

    # is_review_ofs[]
    if reviewed_item and endnote_type in [TYPE_BOOK_SECTION, TYPE_JOURNAL_ARTICLE]:
        is_review_ofs = Document()
        is_review_ofs.set_key_if_not_none("title", reviewed_item)
        is_review_ofs.set_key_if_not_none("rec_type", "Book")
        document.add_items_to_key([is_review_ofs], "is_review_ofs")

    # descriptions[0].value
    if abstract:
        document["descriptions"] = [{"value": abstract, "language": "und"}]

    # archive
    if endnote_type == TYPE_FIGURE and remote_database_provider:
        archive = Document()
        archive["title"] = remote_database_provider
        document.add_items_to_key([archive], "archive")

    # caption
    document.set_key_if_not_none("caption", caption)

    # creators[]
    document.add_creators(primary_creators)
    if endnote_type in [TYPE_BOOK, TYPE_THESIS, TYPE_FILM_OR_BROADCAST]:
        document.add_creators(tertiary_creators)
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST]:
        document.add_creators(subsidiary_creators)
    if custom4:
        document.add_creators(endnote_authors_to_creators(custom4, "person", "rev"))
    if endnote_type == TYPE_FIGURE and remote_database_name:
        document.add_creators(endnote_authors_to_creators(remote_database_name, None, "cph"))

    # edition
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION, TYPE_FILM_OR_BROADCAST, TYPE_WEB_PAGE] and edition:
        document["edition"] = edition

    # extent_pages, extent_volumes
    if endnote_type in [TYPE_BOOK, TYPE_THESIS] and pages:
        document["extent_pages"] = pages.replace("p.", "").strip()
    if endnote_type in [TYPE_BOOK, TYPE_BOOK_SECTION] and extent_volumes:
        document["extent_volumes"] = extent_volumes

    # date_issued, date_issued_first
    if date_year:
        date_issued = ""
        date_issued_first = ""
        orig_index_begin = date_year.find("[")
        orig_index_end = date_year.find("]")
        if orig_index_begin != -1 and orig_index_end != -1:
            date_issued_first = date_year[orig_index_begin + 1: orig_index_end]
            date_issued = date_year.replace("[" + date_issued_first + "]", "").strip()
        else:
            date_issued = date_year.strip()

        if "is_part_ofs" in document:
            if document["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK:
                document["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued)
                document["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first)
            elif "is_part_ofs" in document["is_part_ofs"][0] and document["is_part_ofs"][0]["is_part_ofs"][0]["rec_type"] == constants.DOC_TYPE_BOOK:
                document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued", date_issued)
                document["is_part_ofs"][0]["is_part_ofs"][0].set_key_if_not_none("date_issued_first", date_issued_first)
            else:
                document.set_key_if_not_none("date_issued_first", date_issued_first)
                if rec_type in unpublished_types:
                    document.set_key_if_not_none("date_created", date_issued)
                else:
                    document.set_key_if_not_none("date_issued", date_issued)
        else:
            document.set_key_if_not_none("date_issued_first", date_issued_first)
            if rec_type in unpublished_types:
                document.set_key_if_not_none("date_created", date_issued)
            else:
                document.set_key_if_not_none("date_issued", date_issued)

    # identifiers[]
    identifiers = []
    if accessionnumber:
        identifiers.append(metajson_service.create_identifier("accessionnumber", accessionnumber))
    if callnumber:
        identifiers.append(metajson_service.create_identifier("callnumber", callnumber))
    if doi:
        identifiers.append(metajson_service.create_identifier("doi", doi))
    if identifiers:
        document["identifiers"] = identifiers

    # language
    if language:
        rfc5646 = language_service.convert_unknown_format_to_rfc5646(language)
        if rfc5646:
            document["languages"] = [rfc5646]

    # note
    if endnote_import_note and note:
        document.set_key_with_value_type_in_list("notes", note, "general")
    if endnote_import_research_note and research_notes:
        document.set_key_with_value_type_in_list("notes", research_notes, "user")

    # part_page_begin & part_page_end
    if endnote_type in [TYPE_BOOK_SECTION, TYPE_FIGURE, TYPE_JOURNAL_ARTICLE] and pages:
        hyphen_index = pages.find("-")
        if hyphen_index == -1:
            document["part_page_begin"] = pages.replace("p.", "").strip()
        else:
            document["part_page_begin"] = pages[:hyphen_index].replace("p.", "").strip()
            document["part_page_end"] = pages[hyphen_index+1:].replace("p.", "").strip()

    if endnote_type in [TYPE_JOURNAL_ARTICLE]:
        document.set_key_if_not_none("part_issue", part_number)
    elif endnote_type in [TYPE_FIGURE]:
        document.set_key_if_not_none("part_number", part_number)
    document.set_key_if_not_none("part_section", part_section)
    document.set_key_if_not_none("part_volume", part_volume)

    # resources[0]
    if url is not None:
        resource = Resource()
        resource["rec_type"] = "ResourceRemote"
        resource.set_key_if_not_none("url", url)
        if endnote_type == TYPE_WEB_PAGE:
            resource.set_key_if_not_none("date_last_accessed", part_number)
        else:
            resource.set_key_if_not_none("date_last_accessed", access_date)
        document["resources"] = [resource]

    # subjects[]
    if endnote_import_keywords and keywords:
        for keyword in keywords.split():
            document.set_key_with_value_type_in_list("subjects", keyword, "topic")

    # title, title_alternative, title_abbreviated, title_translated
    document["title"] = title
    if title_alternative:
        document["title_alternatives"] = [{"title": title_alternative}]
    if title_abbreviated:
        document["title_abbreviateds"] = [{"title": title_abbreviated}]

    #logging.debug("# endnote_type: {}".format(endnote_type))
    metajson_service.pretty_print_document(document)
    return document
def ris_txt_lines_to_metajson_list(txt_lines, source, rec_id_prefix, only_first_record):
    document = None
    ris_type = None
    rec_type = None
    is_part_of_rec_type = None
    previous_key = None
    previous_value = None
    for line in txt_lines:
        if line:
            line = line.rstrip('\r\n')
            #logging.debug("line: {}".format(line))

            # multi line management
            if previous_key:
                key = previous_key
                value = previous_value + line
                previous_key = None
                previous_value = None
            else:
                key = line[:2].strip()
                value = line[6:].strip()
            if value.endswith("/") and key not in ["Y1", "PY"]:
                #logging.debug("multi line")
                previous_key = key
                previous_value = value.rstrip('/')
                continue

            if key is None or len(key) == 0:
                # empty line -> continue
                #logging.debug("empty line")
                continue
            elif key == RIS_KEY_BEGIN:
                # record begin with document type -> create document
                # init
                document = Document()
                is_part_of_rec_type = None

                if source:
                    document["rec_source"] = source
                ris_type = value
                rec_type = ris_document_type_to_metajson_document_type[ris_type]
                document["rec_type"] = rec_type
                if ris_type in ris_document_type_to_metajson_document_is_part_of_type:
                    is_part_of_rec_type = ris_document_type_to_metajson_document_is_part_of_type[ris_type]
                    is_part_of = Document()
                    is_part_of["rec_type"] = is_part_of_rec_type
                    document["is_part_ofs"] = [is_part_of]
            elif key == RIS_KEY_END:
                # record end -> return the result
                # verify the is_part_ofs[0]["title"]
                if "is_part_ofs" in document and "title" not in document["is_part_ofs"][0] and "title_abbreviateds" in document["is_part_ofs"][0]:
                    document["is_part_ofs"][0]["title"] = document["is_part_ofs"][0]["title_abbreviateds"][0]["title"]
                    del document["is_part_ofs"][0]["title_abbreviateds"]

                logging.info("# RIS type: {}".format(ris_type))
                metajson_service.pretty_print_document(document)
                yield document
            else:
                # process key value
                #logging.debug("key: {}; value: {}".format(key, value))
                if key == "ID":
                    document["rec_id"] = value
                elif key in ["T1", "TI", "CT"] or (key == "BT" and ris_type in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]):
                    # Title Primary -> title
                    document["title"] = value
                elif key in ["JF", "JO"] or (key == "BT" and ris_type not in [RIS_TYPE_BOOK, RIS_TYPE_UNPB]):
                    # Title Secondary -> is_part_of["title"]
                    document.add_is_part_of_title(value)
                elif key in ["JA", "J1", "J2", "T2"]:
                    # Title Secondary -> is_part_of["title_abbreviateds"][O]["title"]
                    document.add_is_part_of_title_abbreviated(value)
                elif key == "T3":
                    # Title Series
                    document.add_series_title(value)
                elif key in ["A1", "AU"]:
                    document.add_creator(creator_service.formatted_name_to_creator(value, None, "aut"))
                elif key in ["A2", "ED"]:
                    if is_part_of_rec_type:
                        document.add_is_part_of_creator(creator_service.formatted_name_to_creator(value, None, "edt"))
                    else:
                        document.add_creator(creator_service.formatted_name_to_creator(value, None, "edt"))
                elif key == "A3":
                    document.add_series_creator(creator_service.formatted_name_to_creator(value, None, "aut"))
                elif key == "A4":
                    document.add_creator(creator_service.formatted_name_to_creator(value, None, "ctb"))
                elif key in ["PY", "Y1", "DA"]:
                    index_slash = value.find("/")
                    if index_slash != -1:
                        # YYYY/MM/DD/other info (like season)
                        # todo
                        document["date_issued"] = value.strip("/")
                    else:
                        document["date_issued"] = value
                elif key == "SP":
                    document["part_page_begin"] = value
                elif key == "EP":
                    document["part_page_end"] = value
                elif key == "VL":
                    document["part_volume"] = value
                elif key in ["IS", "CP"]:
                    document["part_issue"] = value
                elif key in ["AB", "N2"]:
                    document["descriptions"] = [{"language": "und", "value": value}]
                elif key == "N1":
                    document["notes"] = [{"language": "und", "value": value}]
                elif key == "PB":
                    document.add_item_to_key(value, "publishers")
                elif key == "CY":
                    document.add_item_to_key(value, "publication_places")
                elif key == "RP":
                    document["publication_status"] = value
                elif key == "ET":
                    document["edition"] = value
                elif key == "UR":
                    resource = Resource()
                    resource["url"] = value
                    document.add_item_to_key(resource, "resources")
                elif key == "AN":
                    # Accession Number
                    identifier = metajson_service.create_identifier("accessionnumber", value)
                    document.add_identifier(identifier)
                elif key == "CN":
                    # Call Number
                    identifier = metajson_service.create_identifier("callnumber", value)
                    document.add_identifier(identifier)
                elif key == "DO":
                    # DOI
                    identifier = metajson_service.create_identifier("doi", value)
                    document.add_identifier(identifier)
                elif key == "SN":
                    # ISBN or ISSN ?
                    id_type = None
                    if rec_type in [constants.DOC_TYPE_JOURNALARTICLE, constants.DOC_TYPE_MAGAZINEARTICLE, constants.DOC_TYPE_NEWSPAPERARTICLE, constants.DOC_TYPE_JOURNAL]:
                        id_type = "issn"
                    else:
                        id_type = "isbn"
                    identifier = metajson_service.create_identifier(id_type, value)
                    if is_part_of_rec_type is None:
                        document.add_identifier(identifier)
                    else:
                        document["is_part_ofs"][0].add_identifier(identifier)
                elif key == "CA":
                    document["caption"] = value
                elif key == "DB":
                    # Name of Database -> rec_source ?
                    document["rec_source"] = value
                elif key == "DP":
                    # NDatabase Provider -> rec_source ?
                    document["rec_source"] = value
                elif key == "KW":
                    if "keywords" not in document:
                        document["keywords"] = {"und": []}
                    document["keywords"]["und"].append(value)
                else:
                    logging.debug("Not managed key: {} with value: {}".format(key, value))
Exemple #15
0
def unixref_record_to_metajson(record, source, rec_id_prefix):
    journal_metadata = record.find("./crossref/journal/journal_metadata")
    if journal_metadata is not None:
        journal_title = journal_metadata.find("full_title")
        journal_title_abbreviated = journal_metadata.find("abbrev_title")
        for issn in journal_metadata.findall("issn"):
            if issn.get("media_type") == "print":
                journal_issn = issn
            elif issn.get("media_type") == "electronic":
                journal_eissn = issn

    journal_issue = record.find("./crossref/journal/journal_issue")
    if journal_issue is not None:
        part_issue = journal_issue.find("issue")
        part_volume = journal_issue.find("journal_volume/volume")

    journal_article = record.find("./crossref/journal/journal_article")
    if journal_article is not None:
        titles = journal_article.findall("titles/title")
        creators = journal_article.findall("creators")
        year = journal_article.find("publication_date/year")
        month = journal_article.find("publication_date/month")
        day = journal_article.find("publication_date/day")
        part_page_begin = journal_article.find("pages/first_page")
        part_page_end = journal_article.find("pages/last_page")
        doi = journal_article.find("doi_data/doi")
        resource_url = journal_article.find("doi_data/resource")

    document = Document()
    # todo: how to find the type ?
    document["rec_type"] = "JournalArticle"

    if doi is not None:
        document.set_key_with_value_type_in_list("identifiers", doi.text, "doi")

    if resource_url is not None:
        resource = Resource()
        resource["url"] = resource_url.text
        document["resources"] = [resource]

    if source:
        document["rec_source"] = source

    if titles is not None and len(titles) > 0:
        document["title"] = titles[0].text

    date_issued = year.text
    if month is not None:
        month = month.text
        if len(month) == 1:
            month = "0" + month
        date_issued += "-" + month
    if day is not None:
        day = day.text
        if len(day) == 1:
            day = "0" + day
        date_issued += "-" + day
    document["date_issued"] = date_issued

    if part_issue is not None:
        document["part_issue"] = part_issue.text

    if part_volume is not None:
        document["part_volume"] = part_volume.text

    if part_page_begin is not None:
        document["part_page_begin"] = part_page_begin.text

    if part_page_end is not None:
        document["part_page_end"] = part_page_end.text

    # todo: creator

    is_part_of = Document()
    # todo: how to find the type ?
    is_part_of["rec_type"] = "Journal"

    if journal_title is not None:
        is_part_of["title"] = journal_title.text
    if journal_title_abbreviated is not None:
        is_part_of["title_abbreviateds"] = [{"title":journal_title_abbreviated.text}]
    if journal_issn is not None:
        is_part_of.set_key_with_value_type_in_list("identifiers", journal_issn.text, "issn")
    if journal_eissn is not None:
        is_part_of.set_key_with_value_type_in_list("identifiers", journal_eissn.text, "eissn")

    document["is_part_ofs"] = [is_part_of]

    return document
Exemple #16
0
def summonjson_document_to_metajson(sum_doc, source, rec_id_prefix):
    document = Document()

    # Extract Summon properties
    rec_id = sum_doc["ID"][0].replace("FETCH-", "")
    sum_type = sum_doc["ContentType"][0]
    rec_type = summon_document_type_to_metajson_document_type[sum_type]

    # rec_id, rec_source, rec_type
    document["rec_id"] = rec_id
    if source:
        document["rec_source"] = source
    document["rec_type"] = rec_type

    # languages
    main_language = None
    if "Language" in sum_doc:
        languages = []
        for sum_lang in sum_doc["Language"]:
            lang = language_service.convert_english_to_rfc5646(sum_lang)
            if lang:
                languages.append(lang)
        if languages:
            main_language = languages[0]
            document["languages"] = languages

    # extract summon properties
    creators = extract_creators(sum_doc)
    copyright_statement = extract_value(sum_doc, "Copyright")
    database_id = extract_value(sum_doc, "DBID")
    #logging.debug("DBID: {}".format(database_id))
    database_xml = extract_dict_value(sum_doc, "Database_xml")
    #logging.debug("database_xml: {}".format(database_xml))
    date_issued = extract_date_issued(sum_doc)
    degree = extract_value(sum_doc, "DissertationDegree")
    descriptions = extract_convert_languageValues(sum_doc, "Abstract", main_language)
    edition = extract_value(sum_doc, "Edition")
    extent_pages = extract_value(sum_doc, "PageCount")
    genre = extract_value(sum_doc, "Genre")
    is_part_of_edition = extract_value(sum_doc, "PublicationEdition")
    is_part_of_title = extract_value(sum_doc, "PublicationTitle")
    is_part_of_title_sub = extract_value(sum_doc, "PublicationSubtitle")
    notes = extract_convert_languageValues(sum_doc, "Notes", main_language)
    part_issue = extract_value(sum_doc, "Issue")
    part_page_end = extract_value(sum_doc, "EndPage")
    part_page_begin = extract_value(sum_doc, "StartPage")
    part_volume = extract_value(sum_doc, "Volume")
    peer_reviewed = extract_boolean_value(sum_doc, "IsPeerReviewed")
    publisher = extract_value(sum_doc, "Publisher")
    publication_place = extract_value(sum_doc, "PublicationPlace")
    scholarly = extract_boolean_value(sum_doc, "IsScholarly")
    series_title = extract_value(sum_doc, "PublicationSeriesTitle")
    subject_keywords = extract_value(sum_doc, "Keywords", True)
    subject_agents = convert_creators(sum_doc, "RelatedPersons", None, "person", None)
    subject_topics = extract_value(sum_doc, "SubjectTerms", True)
    table_of_contents = extract_convert_languageValues(sum_doc, "TableOfContents", main_language)
    title = string.strip_html_tags(extract_value(sum_doc, "Title"))
    title_sub = string.strip_html_tags(extract_value(sum_doc, "Subtitle"))

    # identifiers
    has_isbn = False
    has_eissn = False
    identifiers_item = []
    is_part_of_identifiers = []
    for sum_key in summon_identifier_type_to_metajson_identifier_type:
        if sum_key in sum_doc:
            for id_value in sum_doc[sum_key]:
                id_type = summon_identifier_type_to_metajson_identifier_type[sum_key]
                if id_type == "issn":
                    is_part_of_identifiers.append(metajson_service.create_identifier(id_type, id_value))
                elif id_type == "eissn":
                    has_eissn = True
                    is_part_of_identifiers.append(metajson_service.create_identifier(id_type, id_value))
                elif id_type == "isbn":
                    has_isbn = True
                    is_part_of_identifiers.append(metajson_service.create_identifier(id_type, id_value))
                else:
                    identifiers_item.append(metajson_service.create_identifier(id_type, id_value))

    # is_part_of_type determination
    is_part_of_type = None
    if sum_type in summon_document_type_to_metajson_document_is_part_of_type:
        is_part_of_type = summon_document_type_to_metajson_document_is_part_of_type[sum_type]

    elif is_part_of_title and is_part_of_title != title and rec_type not in ["Book", "Journal", "Magazine", "Newspaper", "Periodical"]:
        if has_isbn:
            is_part_of_type = "Book"
        elif has_eissn:
            is_part_of_type = "Journal"
        elif is_part_of_title.lower().find("conference") != -1:
            is_part_of_type = "Book"
        elif is_part_of_title.lower().find("review") or is_part_of_title.lower().find("journal"):
            is_part_of_type = "Journal"
        elif rec_type == "Dataset":
            is_part_of_type = "Periodical"
        else:
            logging.debug("unknown is_part_of_type for rec_type: {}".format(rec_type))

    # is_part_of
    if is_part_of_type:
        is_part_of = Document()
        is_part_of.set_key_if_not_none("rec_type", is_part_of_type)
        is_part_of.set_key_if_not_none("edition", is_part_of_edition)
        is_part_of.add_items_to_key(is_part_of_identifiers, "identifiers")
        is_part_of.set_key_if_not_none("peer_reviewed", peer_reviewed)
        is_part_of.set_key_if_not_none("publishers", [publisher])
        is_part_of.set_key_if_not_none("publication_places", [publication_place])
        is_part_of.set_key_if_not_none("title", is_part_of_title)
        is_part_of.set_key_if_not_none("title_sub", is_part_of_title_sub)

        document.add_items_to_key(identifiers_item, "identifiers")

        document.add_items_to_key([is_part_of], "is_part_ofs")
    else:
        document.set_key_if_not_none("peer_reviewed", peer_reviewed)
        document.set_key_if_not_none("publishers", [publisher])
        document.set_key_if_not_none("publication_places", [publication_place])
        document.add_items_to_key(is_part_of_identifiers, "identifiers")
        document.add_items_to_key(identifiers_item, "identifiers")

    # series
    if series_title:
        series = Document()
        series["rec_type"] = constants.DOC_TYPE_SERIES
        series.set_key_if_not_none("title", series_title)

        document.add_items_to_key([series], "seriess")

    # classificiations
    extract_convert_add_classifications(sum_doc, document, "DEWEY", "ddc")
    extract_convert_add_classifications(sum_doc, document, "Discipline", "discipline")
    extract_convert_add_classifications(sum_doc, document, "NAICS", "NAICS")

    # set properties
    document.set_key_if_not_none("creators", creators)
    document.set_key_if_not_none("copyright_statement", copyright_statement)
    document.set_key_if_not_none("date_issued", date_issued)
    document.set_key_if_not_none("degree", degree)
    document.set_key_if_not_none("descriptions", descriptions)
    document.set_key_if_not_none("edition", edition)
    document.set_key_if_not_none("extent_pages", extent_pages)
    document.set_key_if_not_none("genre", genre)
    document.set_key_if_not_none("notes", notes)
    document.set_key_if_not_none("part_issue", part_issue)
    document.set_key_if_not_none("part_page_end", part_page_end)
    document.set_key_if_not_none("part_page_begin", part_page_begin)
    document.set_key_if_not_none("part_volume", part_volume)
    document.set_key_if_not_none("scholarly", scholarly)
    document.set_key_if_not_none("table_of_contents", table_of_contents)
    document.set_key_if_not_none("title", title)
    document.set_key_if_not_none("title_sub", title_sub)

    # keywords, subjects
    subjects = []
    keywords = {main_language: []}
    if subject_keywords:
        keywords[main_language].extend(subject_keywords)
    if subject_topics:
        keywords[main_language].extend(subject_topics)
    if subject_agents:
        subject = Subject()
        subject["agents"] = subject_agents
        subjects.append(subject)
    if subjects:
        document["subjects"] = subjects
    if keywords[main_language]:
        document["keywords"] = keywords

    logging.info("# Summon ContentType: {}".format(sum_type))
    metajson_service.pretty_print_document(document)
    return document