def parse_record(self, row: Dict[str, Any]) -> Optional[ContainerEntity]: """ row is a python dict (parsed from JSON). returns a ContainerEntity (or None if invalid or couldn't parse) """ name = clean_str(row.get("name")) if name and name.endswith("."): name = name[:-1] if not name: # Name is required (by schema) return None extra = dict() for k in ( "urls", "webarchive_urls", "country", "sherpa_romeo", "ezb", "szczepanski", "doaj", "languages", "ia", "scielo", "kbart", "publisher_type", "platform", ): if row["extra"].get(k): extra[k] = row["extra"][k] container_type = None if "proceedings" in name.lower(): container_type = "proceedings" elif "journal " in name.lower(): container_type = "journal" if row["extra"].get("issnp"): row["extra"]["issnp"] = row["extra"]["issnp"].upper() if row["extra"].get("issne"): row["extra"]["issne"] = row["extra"]["issne"].upper() ce = ContainerEntity( issnl=row["issnl"], issnp=row["extra"].get("issnp"), issne=row["extra"].get("issne"), ident=row["ident"], name=name, container_type=container_type, publisher=clean_str(row.get("publisher")), wikidata_qid=row.get("wikidata_qid"), extra=extra, ) return ce
def doaj_contribs( self, authors: List[dict]) -> List[fatcat_openapi_client.ReleaseContrib]: """ bibjson.author { affiliation (string, optional), name (string), orcid_id (string, optional) } """ contribs = [] index = 0 for author in authors: if not author.get("name"): continue creator_id = None orcid = clean_orcid(author.get("orcid_id")) if orcid: creator_id = self.lookup_orcid(orcid) contribs.append( fatcat_openapi_client.ReleaseContrib( raw_name=author.get("name"), role="author", index=index, creator_id=creator_id, raw_affiliation=clean_str(author.get("affiliation")), )) index += 1 return contribs
def dblp_contrib_single(self, elem: Any) -> fatcat_openapi_client.ReleaseContrib: """ In the future, might try to implement creator key-ificiation and lookup here. Example rows: <author>Michael H. Böhlen</author> <author orcid="0000-0002-4354-9138">Nicolas Heist</author> <author orcid="0000-0001-9108-4278">Jens Lehmann 0001</author> """ creator_id = None extra = None raw_name = clean_str(elem.text) # remove number in author name, if present if raw_name.split()[-1].isdigit(): raw_name = ' '.join(raw_name.split()[:-1]) if elem.get('orcid'): orcid = clean_orcid(elem['orcid']) if orcid: creator_id = self.lookup_orcid(orcid) if not creator_id: extra = dict(orcid=orcid) return fatcat_openapi_client.ReleaseContrib( raw_name=raw_name, creator_id=creator_id, extra=extra, )
def parse_record(self, obj: Dict[str, Any]) -> Optional[CreatorEntity]: """ obj is a python dict (parsed from json). returns a CreatorEntity """ if "person" not in obj: return False name = obj["person"]["name"] if not name: return None extra = None given = value_or_none(name.get("given-names")) sur = value_or_none(name.get("family-name")) display = value_or_none(name.get("credit-name")) if display is None: # TODO: sorry human beings if given and sur: display = "{} {}".format(given, sur) elif sur: display = sur elif given: display = given orcid = obj["orcid-identifier"]["path"] if not self.is_orcid(orcid): sys.stderr.write("Bad ORCID: {}\n".format(orcid)) return None display = clean_str(display) if not display: # must have *some* name return None ce = CreatorEntity( orcid=orcid, given_name=clean_str(given), surname=clean_str(sur), display_name=display, extra=extra, ) return ce
def parse_record(self, row: Dict[str, Any]) -> Optional[ContainerEntity]: """ row is a python dict (parsed from JSON). returns a ContainerEntity (or None if invalid or couldn't parse) """ dblp_prefix = row.get("key") or row.get("dblp_prefix") assert dblp_prefix assert row["title"] container_type = None if dblp_prefix.startswith("conf/"): container_type = "conference-series" elif dblp_prefix.startswith("journals/"): container_type = "journal" elif dblp_prefix.startswith("series/"): container_type = "book-series" issnl = None for issn in row.get("issns", []): issnl = self.issn2issnl(issn) if issnl: break extra: Dict[str, Any] = { "dblp": { "prefix": dblp_prefix, }, } if row.get("homepage_url"): extra["urls"] = [row["homepage_url"]] if row.get("acronym"): extra["acronym"] = row["acronym"] ce = fatcat_openapi_client.ContainerEntity( name=clean_str(row["title"]), container_type=container_type, issnl=issnl, wikidata_qid=row.get("wikidata_qid"), extra=extra, ) return ce
def parse_record(self, row): """ row is a python dict (parsed from JSON). returns a ContainerEntity (or None if invalid or couldn't parse) """ dblp_prefix = row.get('key') or row.get('dblp_prefix') assert dblp_prefix assert row['title'] container_type = None if dblp_prefix.startswith('conf/'): container_type = "conference-series" elif dblp_prefix.startswith('journals/'): container_type = "journal" elif dblp_prefix.startswith('series/'): container_type = "book-series" issnl = None for issn in row.get('issns', []): issnl = self.issn2issnl(issn) if issnl: break extra = { 'dblp': { 'prefix': dblp_prefix, }, } if row.get('homepage_url'): extra['urls'] = [row['homepage_url']] if row.get('acronym'): extra['acronym'] = row['acronym'] ce = fatcat_openapi_client.ContainerEntity( name=clean_str(row['title']), container_type=container_type, issnl=issnl, wikidata_qid=row.get('wikidata_qid'), extra=extra, ) return ce
def do_contribs(obj_list: List[Dict[str, Any]], ctype: str) -> List[ReleaseContrib]: contribs = [] for i, am in enumerate(obj_list): creator_id = None if "ORCID" in am.keys(): creator_id = self.lookup_orcid(am["ORCID"].split("/")[-1]) # Sorry humans :( if am.get("given") and am.get("family"): raw_name: Optional[str] = "{} {}".format( am["given"], am["family"]) elif am.get("family"): raw_name = am["family"] else: # TODO: can end up empty raw_name = am.get("name") or am.get("given") extra: Dict[str, Any] = dict() if ctype == "author": index: Optional[int] = i else: index = None raw_affiliation = None affiliation_list = am.get("affiliation") or [] # TODO: currently requiring a "name" in all affiliations. Could # add ROR support (via identifier) in the near future affiliation_list = [a for a in affiliation_list if "name" in a] if affiliation_list and len(affiliation_list) > 0: raw_affiliation = affiliation_list[0]["name"] if len(affiliation_list) > 1: # note: affiliation => more_affiliations extra["more_affiliations"] = [ clean_str(a["name"]) for a in affiliation_list[1:] ] if am.get("sequence") and am.get("sequence") != "additional": extra["seq"] = clean_str(am.get("sequence")) assert ctype in ("author", "editor", "translator") raw_name = clean_str(raw_name) # TODO: what if 'raw_name' is None? contribs.append( ReleaseContrib( creator_id=creator_id, index=index, raw_name=raw_name, given_name=clean_str(am.get("given")), surname=clean_str(am.get("family")), raw_affiliation=clean_str(raw_affiliation), role=ctype, extra=extra or None, )) return contribs
def doaj_abstracts( self, bibjson: dict) -> List[fatcat_openapi_client.ReleaseAbstract]: text = clean_str(bibjson.get("abstract")) if not text or len(text) < 10: return [] if len(text) > MAX_ABSTRACT_LENGTH: text = text[:MAX_ABSTRACT_LENGTH] + " [...]" lang = detect_text_lang(text) abstract = fatcat_openapi_client.ReleaseAbstract( mimetype="text/plain", content=text, lang=lang, ) return [ abstract, ]
def parse_record(self, row: Dict[str, Any]) -> Optional[ContainerEntity]: """ row is a python dict (parsed from JSON). returns a ContainerEntity (or None if invalid or couldn't parse) """ if not row.get("name"): # Name is required (by schema) return None extra = dict() for key in ( "issne", "issnp", "languages", "country", "urls", "abbrev", "coden", "aliases", "original_name", "first_year", "last_year", "platform", "default_license", "road", "mimetypes", "sherpa_romeo", "kbart", ): if row.get(key): extra[key] = row[key] # TODO: not including for now: norwegian, dois/crossref, ia extra_doaj = dict() if row.get("doaj"): if row["doaj"].get("as_of"): extra_doaj["as_of"] = row["doaj"]["as_of"] if row["doaj"].get("works"): extra_doaj["works"] = row["doaj"]["works"] if extra_doaj: extra["doaj"] = extra_doaj extra_ia = dict() # TODO: would like an ia.longtail_ia flag if row.get("sim"): # NB: None case of the .get() here is blech, but othrwise # extra['ia'].get('sim') would be false-y, breaking 'any_ia_sim' later on extra_ia["sim"] = { "year_spans": row["sim"].get("year_spans"), } if extra_ia: extra["ia"] = extra_ia name = clean_str(row.get("name")) if not name: return None ce = ContainerEntity( issnl=row["issnl"], issne=row.get("issne"), issnp=row.get("issnp"), container_type=None, # TODO name=name, publisher=clean_str(row.get("publisher")), wikidata_qid=None, # TODO extra=extra, ) return ce
def parse_record(self, obj: Dict[str, Any]) -> Optional[ReleaseEntity]: """ obj is a python dict (parsed from json). returns a ReleaseEntity """ # Ways to be out of scope (provisionally) # journal-issue and journal-volume map to None, but allowed for now if obj.get("type") in ( None, "journal", "proceedings", "standard-series", "report-series", "book-series", "book-set", "book-track", "proceedings-series", ): self.counts["skip-release-type"] += 1 return None # Do require the 'title' keys to exist, as release entities do if ("title" not in obj) or (not obj["title"]): self.counts["skip-blank-title"] += 1 return None release_type = self.map_release_type(obj["type"]) # contribs def do_contribs(obj_list: List[Dict[str, Any]], ctype: str) -> List[ReleaseContrib]: contribs = [] for i, am in enumerate(obj_list): creator_id = None if "ORCID" in am.keys(): creator_id = self.lookup_orcid(am["ORCID"].split("/")[-1]) # Sorry humans :( if am.get("given") and am.get("family"): raw_name: Optional[str] = "{} {}".format( am["given"], am["family"]) elif am.get("family"): raw_name = am["family"] else: # TODO: can end up empty raw_name = am.get("name") or am.get("given") extra: Dict[str, Any] = dict() if ctype == "author": index: Optional[int] = i else: index = None raw_affiliation = None affiliation_list = am.get("affiliation") or [] # TODO: currently requiring a "name" in all affiliations. Could # add ROR support (via identifier) in the near future affiliation_list = [a for a in affiliation_list if "name" in a] if affiliation_list and len(affiliation_list) > 0: raw_affiliation = affiliation_list[0]["name"] if len(affiliation_list) > 1: # note: affiliation => more_affiliations extra["more_affiliations"] = [ clean_str(a["name"]) for a in affiliation_list[1:] ] if am.get("sequence") and am.get("sequence") != "additional": extra["seq"] = clean_str(am.get("sequence")) assert ctype in ("author", "editor", "translator") raw_name = clean_str(raw_name) # TODO: what if 'raw_name' is None? contribs.append( ReleaseContrib( creator_id=creator_id, index=index, raw_name=raw_name, given_name=clean_str(am.get("given")), surname=clean_str(am.get("family")), raw_affiliation=clean_str(raw_affiliation), role=ctype, extra=extra or None, )) return contribs contribs = do_contribs(obj.get("author", []), "author") contribs.extend(do_contribs(obj.get("editor", []), "editor")) contribs.extend(do_contribs(obj.get("translator", []), "translator")) # container issn = obj.get("ISSN", [None])[0] issnl = self.issn2issnl(issn) container_id = None if issnl: container_id = self.lookup_issnl(issnl) publisher = clean_str(obj.get("publisher")) container_name = obj.get("container-title") if container_name: container_name = clean_str(container_name[0], force_xml=True) if not container_name: container_name = None if (container_id is None and self.create_containers and (issnl is not None) and container_name): ce = fatcat_openapi_client.ContainerEntity( issnl=issnl, publisher=publisher, container_type=self.map_container_type(release_type), name=container_name, ) ce_edit = self.create_container(ce) container_id = ce_edit.ident self._issnl_id_map[issnl] = container_id # license slug license_slug = None license_extra = [] for lic in obj.get("license", []): if lic["content-version"] not in ("vor", "unspecified"): continue slug = lookup_license_slug(lic["URL"]) if slug: license_slug = slug if "start" in lic: lic["start"] = lic["start"]["date-time"] license_extra.append(lic) # references refs = [] for i, rm in enumerate(obj.get("reference", [])): try: year: Optional[int] = int(rm.get("year")) # TODO: will need to update/config in the future! # NOTE: are there crossref works with year < 100? if year is not None: if year > 2025 or year < 100: year = None except (TypeError, ValueError): year = None ref_extra: Dict[str, Any] = dict() key = rm.get("key") if key and key.startswith(obj["DOI"].upper()): key = key.replace(obj["DOI"].upper() + "-", "") key = key.replace(obj["DOI"].upper(), "") ref_container_name = rm.get("volume-title") if not ref_container_name: ref_container_name = rm.get("journal-title") elif rm.get("journal-title"): ref_extra["journal-title"] = rm["journal-title"] if rm.get("DOI"): ref_extra["doi"] = rm.get("DOI").lower() author = clean_str(rm.get("author")) if author: ref_extra["authors"] = [author] for k in ( "editor", "edition", "authority", "version", "genre", "url", "event", "issue", "volume", "date", "accessed_date", "issued", "page", "medium", "collection_title", "chapter_number", "unstructured", "series-title", "volume-title", ): if clean_str(rm.get(k)): ref_extra[k] = clean_str(rm[k]) refs.append( fatcat_openapi_client.ReleaseRef( index=i, # doing lookups would be a second import pass target_release_id=None, key=key, year=year, container_name=clean_str(ref_container_name), title=clean_str(rm.get("article-title")), locator=clean_str(rm.get("first-page")), # TODO: just dump JSON somewhere here? extra=ref_extra or None, )) # abstracts abstracts = [] abstract = clean_str(obj.get("abstract")) if abstract and len(abstract) > 10: abstracts.append( fatcat_openapi_client.ReleaseAbstract( mimetype="application/xml+jats", content=abstract)) # extra fields extra: Dict[str, Any] = dict() extra_crossref: Dict[str, Any] = dict() # top-level extra keys if not container_id: if obj.get("container-title"): extra["container_name"] = container_name for key in "group-title": val = obj.get(key) if val: if type(val) == list: val = val[0] if type(val) == str: val = clean_str(val) if val: extra[key] = clean_str(val) else: extra[key] = val # crossref-nested extra keys for key in ("subject", "type", "alternative-id", "archive", "funder"): val = obj.get(key) if val: if type(val) == str: extra_crossref[key] = clean_str(val) else: extra_crossref[key] = val if license_extra: extra_crossref["license"] = license_extra if len(obj["title"]) > 1: aliases = [clean_str(t) for t in obj["title"][1:]] aliases = [t for t in aliases if t] if aliases: extra["aliases"] = aliases # ISBN isbn13 = None for raw in obj.get("ISBN", []): # TODO: convert if not ISBN-13 format if len(raw) == 17: isbn13 = raw break # release status if obj["type"] in ( "journal-article", "conference-proceeding", "book", "dissertation", "book-chapter", ): release_stage: Optional[str] = "published" else: # unknown release_stage = None # filter out unreasonably huge releases if len(abstracts) > 100: self.counts["skip-huge-abstracts"] += 1 return None if len(contribs) > 2000: self.counts["skip-huge-contribs"] += 1 return None if len(refs) > 5000: self.counts["skip-huge-refs"] += 1 return None # release date parsing is amazingly complex raw_date = obj["issued"]["date-parts"][0] if not raw_date or not raw_date[0]: # got some NoneType, even though at least year is supposed to be set release_year = None release_date = None elif len(raw_date) == 3: release_year = raw_date[0] release_date = datetime.date(year=raw_date[0], month=raw_date[1], day=raw_date[2]) else: # sometimes only the year is included, not the full date release_year = raw_date[0] release_date = None original_title: Optional[str] = None if obj.get("original-title"): ot = obj.get("original-title") if ot is not None: original_title = clean_str(ot[0], force_xml=True) title: Optional[str] = None if obj.get("title"): title = clean_str(obj["title"][0], force_xml=True) if not title or len(title) <= 1: # title can't be just a single character self.counts["skip-blank-title"] += 1 return None doi = clean_doi(obj["DOI"].lower()) if not doi: self.counts["skip-bad-doi"] += 1 return None subtitle = None if obj.get("subtitle"): subtitle = clean_str(obj["subtitle"][0], force_xml=True) if not subtitle or len(subtitle) <= 1: # subtitle can't be just a single character subtitle = None if extra_crossref: extra["crossref"] = extra_crossref re = ReleaseEntity( work_id=None, container_id=container_id, title=title, subtitle=subtitle, original_title=original_title, release_type=release_type, release_stage=release_stage, release_date=release_date, release_year=release_year, publisher=publisher, ext_ids=fatcat_openapi_client.ReleaseExtIds( doi=doi, isbn13=isbn13, ), volume=clean_str(obj.get("volume")), issue=clean_str(obj.get("issue")), pages=clean_str(obj.get("page")), language=clean_str(obj.get("language")), license_slug=license_slug, extra=extra or None, abstracts=abstracts or None, contribs=contribs or None, refs=refs or None, ) return re
def parse_jalc_persons(raw_persons: List[Any]) -> List[ReleaseContrib]: """ For the most part, JALC DC names are in either japanese or english. The two common patterns are a list alternating between the two (in which case the names are translations), or all in one language or the other. Because dublin core is a projection tossing away a bunch of context, the other cases are hard to disambiguate. There are also some cases with Korean and other languages mixed in. This crude method doesn't handle everything right; it tries to just get the two common patterns correct. Sorry humans! Edge cases for this function: - 10.11316/jpsgaiyo.56.1.4.0_757_3 <= all english, some japanese, works - 10.14988/pa.2017.0000013531 <= complex, not japanese/english, mixed - 10.15036/arerugi.62.1407_1 <= one japanese, two english; fails - 10.14988/pa.2017.0000007327 <= ambiguous; translator in jpn/eng """ persons = [] # first parse out into language-agnostic dics for raw in raw_persons: name = raw.find("name") or None if name: name = clean_str(name.get_text().replace("\n", " ")) surname = raw.find("familyName") or None if surname: surname = clean_str(surname.get_text().replace("\n", " ")) given_name = raw.find("givenName") or None if given_name: given_name = clean_str(given_name.get_text().replace("\n", " ")) lang = "en" if is_cjk(name): lang = "ja" if lang == "en" and surname and given_name: # english names order is flipped name = "{} {}".format(given_name, surname) rc = ReleaseContrib(raw_name=name, surname=surname, given_name=given_name, role="author") # add an extra hint field; won't end up in serialized object rc._lang = lang persons.append(rc) if not persons: return [] if all([p._lang == "en" for p in persons]) or all([p._lang == "ja" for p in persons]): # all english names, or all japanese names return persons # for debugging # if len([1 for p in persons if p._lang == 'en']) != len([1 for p in persons if p._lang == 'ja']): # print("INTERESTING: {}".format(persons[0])) start_lang = persons[0]._lang contribs = [] for p in persons: if p._lang == start_lang: contribs.append(p) else: if p._lang == "en" and contribs[-1]._lang == "ja": eng = p jpn = contribs[-1] elif p._lang == "ja" and contribs[-1]._lang == "en": eng = contribs[-1] jpn = p else: # give up and just add as another author contribs.append(p) continue eng.extra = { "original_name": { "lang": jpn._lang, "raw_name": jpn.raw_name, "given_name": jpn.given_name, "surname": jpn.surname, }, } contribs[-1] = eng return contribs
def parse_record(self, record: Any) -> Optional[ReleaseEntity]: """ record is a beautiful soup object returns a ReleaseEntity, or None In JALC metadata, both English and Japanese records are given for most fields. """ extra: Dict[str, Any] = dict() extra_jalc: Dict[str, Any] = dict() titles = record.find_all("title") if not titles: return None title = titles[0].get_text().replace("\n", " ").strip() original_title = None if title.endswith("."): title = title[:-1] if len(titles) > 1: original_title = titles[1].get_text().replace("\n", " ").strip() if original_title.endswith("."): original_title = original_title[:-1] doi = None if record.doi: doi = clean_doi(record.doi.string.strip().lower()) # TODO: following code is redundant with clean_doi() if not doi: return None if doi.startswith("http://dx.doi.org/"): doi = doi.replace("http://dx.doi.org/", "") elif doi.startswith("https://dx.doi.org/"): doi = doi.replace("https://dx.doi.org/", "") elif doi.startswith("http://doi.org/"): doi = doi.replace("http://doi.org/", "") elif doi.startswith("https://doi.org/"): doi = doi.replace("https://doi.org/", "") if not (doi.startswith("10.") and "/" in doi): sys.stderr.write("bogus JALC DOI: {}\n".format(doi)) doi = None if not doi: return None people = record.find_all("Person") contribs = parse_jalc_persons(people) for i, contrib in enumerate(contribs): if contrib.raw_name != "et al.": contrib.index = i release_year = None release_date = None date = record.date or None if date: date = date.string if len(date) == 10: release_date_date = datetime.datetime.strptime( date["completed-date"], DATE_FMT).date() release_year = release_date_date.year release_date = release_date_date.isoformat() elif len(date) == 4 and date.isdigit(): release_year = int(date) pages = None if record.startingPage and record.startingPage.string.strip(): pages = record.startingPage.string.strip() if record.endingPage and record.endingPage.string.strip(): pages = "{}-{}".format(pages, record.endingPage.string.strip()) # double check to prevent "-" as pages if pages and pages.strip() == "-": pages = None volume = None if record.volume: volume = record.volume.string issue = None if record.number: # note: number/issue transform issue = record.number.string # container issn = None issn_list = record.find_all("issn") if issn_list: # if we wanted the other ISSNs, would also need to uniq the list. # But we only need one to lookup ISSN-L/container issn = issn_list[0].string if issn: issnl = self.issn2issnl(issn) else: issnl = None container_id = None if issnl: container_id = self.lookup_issnl(issnl) publisher = None container_name = None container_extra: Dict[str, Any] = dict() if record.publicationName: pubs = [ p.get_text().replace("\n", " ").strip() for p in record.find_all("publicationName") if p.get_text() ] pubs = [clean_str(p) for p in pubs if p] assert pubs if len(pubs) > 1 and pubs[0] == pubs[1]: pubs = [pubs[0]] if len(pubs) > 1 and is_cjk(pubs[0]): # eng/jpn ordering is not reliable pubs = [pubs[1], pubs[0]] container_name = clean_str(pubs[0]) if len(pubs) > 1: container_extra["original_name"] = clean_str(pubs[1]) if record.publisher: pubs = [ p.get_text().replace("\n", " ").strip() for p in record.find_all("publisher") if p.get_text() ] pubs = [p for p in pubs if p] if len(pubs) > 1 and pubs[0] == pubs[1]: pubs = [pubs[0]] if len(pubs) > 1 and is_cjk(pubs[0]): # ordering is not reliable pubs = [pubs[1], pubs[0]] if pubs: publisher = clean_str(pubs[0]) if len(pubs) > 1: container_extra["publisher_aliases"] = pubs[1:] if (container_id is None and self.create_containers and (issnl is not None) and container_name): # name, type, publisher, issnl # extra: issnp, issne, original_name, languages, country container_extra["country"] = "jp" container_extra["languages"] = ["ja"] ce = fatcat_openapi_client.ContainerEntity( name=container_name, container_type="journal", publisher=publisher, issnl=issnl, extra=(container_extra or None), ) ce_edit = self.create_container(ce) container_id = ce_edit.ident # short-cut future imports in same batch self._issnl_id_map[issnl] = container_id # the vast majority of works are in japanese # TODO: any indication when *not* in japanese? lang = "ja" # reasonable default for this collection release_type = "article-journal" # extra: # translation_of # aliases # container_name # group-title # always put at least an empty dict here to indicate the DOI registrar # (informally) extra["jalc"] = extra_jalc title = clean_str(title) if not title: return None re = ReleaseEntity( work_id=None, title=title, original_title=clean_str(original_title), release_type=release_type, release_stage="published", release_date=release_date, release_year=release_year, ext_ids=fatcat_openapi_client.ReleaseExtIds(doi=doi, ), volume=volume, issue=issue, pages=pages, publisher=publisher, language=lang, # license_slug container_id=container_id, contribs=contribs, extra=extra, ) return re
def parse_datacite_creators( self, creators: List[Dict[str, Any]], role: str = "author", set_index: bool = True, doi: Optional[str] = None, ) -> List[ReleaseContrib]: """ Parses a list of creators into a list of ReleaseContrib objects. Set set_index to False, if the index contrib field should be left blank. The doi parameter is only used for debugging. """ # Contributors. Many nameIdentifierSchemes, we do not use (yet): # "attributes.creators[].nameIdentifiers[].nameIdentifierScheme": # ["LCNA", "GND", "email", "NAF", "OSF", "RRID", "ORCID", # "SCOPUS", "NRCPID", "schema.org", "GRID", "MGDS", "VIAF", "JACoW-ID"]. contribs: List[ReleaseContrib] = [] # Names, that should be ignored right away. name_blocklist = set(("Occdownload Gbif.Org", )) i: Optional[int] = 0 for c in creators: if not set_index: i = None nameType = c.get("nameType", "") or "" if nameType in ("", "Personal"): creator_id = None for nid in c.get("nameIdentifiers", []) or []: if not isinstance(nid, dict): # see: fatcat-workers/issues/44035/ print( "unexpected nameIdentifiers, expected list of dicts, got: {}" .format(nid), file=sys.stderr, ) continue name_scheme = nid.get("nameIdentifierScheme", "") or "" if not name_scheme.lower() == "orcid": continue orcid = nid.get("nameIdentifier") or "" orcid = orcid.replace("https://orcid.org/", "") if not orcid: continue creator_id = self.lookup_orcid(orcid) # TODO(martin): If creator_id is None, should we create creators? # If there are multiple affiliation strings, use the first one. affiliations = c.get("affiliation", []) or [] raw_affiliation = None if len(affiliations) == 0: raw_affiliation = None else: raw_affiliation = clean_str(affiliations[0]) name = c.get("name") given_name = c.get("givenName") surname = c.get("familyName") if name: name = clean_str(name) if not any((name, given_name, surname)): continue if not name: name = "{} {}".format(given_name or "", surname or "").strip() if name in name_blocklist: continue if name.lower() in UNKNOWN_MARKERS_LOWER: continue # Unpack name, if we have an index form (e.g. 'Razis, Panos A') into 'Panos A razis'. if name: name = index_form_to_display_name(name) if given_name: given_name = clean_str(given_name) surname = clean_str(surname) # Perform a final assertion that name does not reduce to zero # (e.g. whitespace only name). if name: name = name.strip() if not name: continue if raw_affiliation == "": continue extra = None # "DataManager", "DataCurator", "ContactPerson", "Distributor", # "RegistrationAgency", "Sponsor", "Researcher", # "RelatedPerson", "ProjectLeader", "Editor", "Other", # "ProjectMember", "Funder", "RightsHolder", "DataCollector", # "Supervisor", "Producer", "HostingInstitution", "ResearchGroup" contributorType = c.get("contributorType", "") or "" if contributorType: extra = {"type": contributorType} rc = fatcat_openapi_client.ReleaseContrib( creator_id=creator_id, index=i, raw_name=name, given_name=given_name, surname=surname, role=role, raw_affiliation=raw_affiliation, extra=extra, ) # Filter out duplicates early. if not contributor_list_contains_contributor(contribs, rc): contribs.append(rc) if i is not None: i += 1 elif nameType == "Organizational": name = c.get("name", "") or "" if name in UNKNOWN_MARKERS: continue if len(name) < 3: continue extra = {"organization": name} contribs.append( fatcat_openapi_client.ReleaseContrib(index=i, extra=extra)) if i is not None: i += 1 else: print("[{}] unknown name type: {}".format(doi, nameType), file=sys.stderr) return contribs
def parse_record(self, article: Any) -> Optional[ReleaseEntity]: journal_meta = article.front.find("journal-meta") article_meta = article.front.find("article-meta") extra: Dict[str, Any] = dict() extra_jstor: Dict[str, Any] = dict() release_type = JSTOR_TYPE_MAP.get(article["article-type"]) title = article_meta.find("article-title") if title and title.get_text(): title = title.get_text().replace("\n", " ").strip() elif title and not title.get_text(): title = None if (not title and release_type and release_type.startswith("review") and article_meta.product.source): title = "Review: {}".format( article_meta.product.source.replace("\n", " ").get_text()) if not title: return None if title.endswith("."): title = title[:-1] if "[Abstract]" in title: # TODO: strip the "[Abstract]" bit? release_type = "abstract" elif "[Editorial" in title: release_type = "editorial" elif "[Letter" in title: release_type = "letter" elif "[Poem" in title or "[Photograph" in title: release_type = None if title.startswith("[") and title.endswith("]"): # strip brackets if that is all that is there (eg, translation or non-english) title = title[1:-1] # JSTOR journal-id journal_ids = [j.string for j in journal_meta.find_all("journal-id")] if journal_ids: extra_jstor["journal_ids"] = journal_ids journal_title = journal_meta.find("journal-title").get_text().replace( "\n", " ") publisher = journal_meta.find("publisher-name").get_text().replace( "\n", " ") issn = journal_meta.find("issn") if issn: issn = issn.string if len(issn) == 8: issn = "{}-{}".format(issn[0:4], issn[4:8]) else: assert len(issn) == 9 issnl = self.issn2issnl(issn) container_id = None if issnl: container_id = self.lookup_issnl(issnl) # create container if it doesn't exist if (container_id is None and self.create_containers and (issnl is not None) and journal_title): ce = fatcat_openapi_client.ContainerEntity( issnl=issnl, publisher=publisher, container_type=self.map_container_type(release_type), name=clean_str(journal_title, force_xml=True), ) ce_edit = self.create_container(ce) container_id = ce_edit.ident self._issnl_id_map[issnl] = container_id doi = article_meta.find("article-id", {"pub-id-type": "doi"}) if doi: doi = clean_doi(doi.string.lower()) else: doi = None jstor_id = article_meta.find("article-id", {"pub-id-type": "jstor"}) if jstor_id: jstor_id = jstor_id.string.strip() if not jstor_id and doi: assert doi.startswith("10.2307/") jstor_id = doi.replace("10.2307/", "") assert jstor_id and int(jstor_id) contribs = [] cgroup = article_meta.find("contrib-group") if cgroup: for c in cgroup.find_all("contrib"): given = c.find("given-names") if given: given = clean_str(given.get_text().replace("\n", " ")) surname = c.find("surname") if surname: surname = clean_str(surname.get_text().replace("\n", " ")) raw_name = c.find("string-name") if raw_name: raw_name = clean_str(raw_name.get_text().replace( "\n", " ")) if not raw_name: if given and surname: raw_name = "{} {}".format(given, surname) elif surname: raw_name = surname role = JSTOR_CONTRIB_MAP.get(c.get("contrib-type", "author")) if not role and c.get("contrib-type"): sys.stderr.write("NOT IN JSTOR_CONTRIB_MAP: {}\n".format( c["contrib-type"])) contribs.append( fatcat_openapi_client.ReleaseContrib( role=role, raw_name=raw_name, given_name=given, surname=surname, )) for i, contrib in enumerate(contribs): if contrib.raw_name != "et al.": contrib.index = i release_year = None release_date = None pub_date = article_meta.find("pub-date") if pub_date and pub_date.year: release_year = int(pub_date.year.string) if pub_date.month and pub_date.day: release_date = datetime.date(release_year, int(pub_date.month.string), int(pub_date.day.string)) if release_date.day == 1 and release_date.month == 1: # suspect jan 1st dates get set by JSTOR when actual # date not known (citation needed), so drop them release_date = None volume = None if article_meta.volume: volume = article_meta.volume.string or None issue = None if article_meta.issue: issue = article_meta.issue.string or None pages = None if article_meta.find("page-range"): pages = article_meta.find("page-range").string elif article_meta.fpage: pages = article_meta.fpage.string language = None cm = article_meta.find("custom-meta") if cm.find("meta-name").string == "lang": language = cm.find("meta-value").string.split()[0] language = LANG_MAP_MARC.get(language) if not language: warnings.warn("MISSING MARC LANG: {}".format( cm.find("meta-value").string)) # JSTOR issue-id if article_meta.find("issue-id"): issue_id = clean_str(article_meta.find("issue-id").string) if issue_id: extra_jstor["issue_id"] = issue_id # everything in JSTOR is published release_stage = "published" # extra: # withdrawn_date # translation_of # subtitle # aliases # container_name # group-title # pubmed: retraction refs if extra_jstor: extra["jstor"] = extra_jstor re = fatcat_openapi_client.ReleaseEntity( # work_id title=title, # original_title release_type=release_type, release_stage=release_stage, release_date=release_date, release_year=release_year, ext_ids=fatcat_openapi_client.ReleaseExtIds( doi=doi, jstor=jstor_id, ), volume=volume, issue=issue, pages=pages, publisher=publisher, language=language, # license_slug # content, mimetype, lang # abstracts=abstracts, contribs=contribs, # key, year, container_name, title, locator # extra: volume, authors, issue, publisher, identifiers # refs=refs, # name, type, publisher, issnl # extra: issnp, issne, original_name, languages, country container_id=container_id, extra=extra or None, ) return re
def parse_record(self, obj: Dict[str, Any]) -> Optional[ReleaseEntity]: """ bibjson { abstract (string, optional), author (Array[bibjson.author], optional), identifier (Array[bibjson.identifier]), journal (bibjson.journal, optional), keywords (Array[string], optional), link (Array[bibjson.link], optional), month (string, optional), subject (Array[bibjson.subject], optional), title (string), year (string, optional) } bibjson.journal { country (string, optional), end_page (string, optional), language (Array[string], optional), license (Array[bibjson.journal.license], optional), number (string, optional), publisher (string, optional), start_page (string, optional), title (string, optional), volume (string, optional) } """ if not obj or not isinstance(obj, dict) or "bibjson" not in obj: self.counts["skip-empty"] += 1 return None bibjson = obj["bibjson"] title = clean_str(bibjson.get("title"), force_xml=True) if not title: self.counts["skip-title"] += 1 return False container_name = clean_str(bibjson["journal"]["title"]) container_id = None # NOTE: 'issns' not documented in API schema for issn in bibjson["journal"]["issns"]: issnl = self.issn2issnl(issn) if issnl: container_id = self.lookup_issnl(issnl) if container_id: # don't store container_name when we have an exact match container_name = None break volume = clean_str(bibjson["journal"].get("volume")) # NOTE: this schema seems to use "number" as "issue number" issue = clean_str(bibjson["journal"].get("number")) publisher = clean_str(bibjson["journal"].get("publisher")) try: release_year: Optional[int] = int(bibjson.get("year")) except (TypeError, ValueError): release_year = None release_month = parse_month(clean_str(bibjson.get("month"))) # block bogus far-future years/dates if release_year is not None and (release_year > (self.this_year + 5) or release_year < 1000): release_month = None release_year = None license_slug = self.doaj_license_slug( bibjson["journal"].get("license")) country = parse_country_name(bibjson["journal"].get("country")) language = None for raw in bibjson["journal"].get("language") or []: language = parse_lang_name(raw) if language: break # pages # NOTE: error in API docs? seems like start_page not under 'journal' object start_page = clean_str( bibjson["journal"].get("start_page")) or clean_str( bibjson.get("start_page")) end_page = clean_str(bibjson["journal"].get("end_page")) or clean_str( bibjson.get("end_page")) pages: Optional[str] = None if start_page and end_page: pages = f"{start_page}-{end_page}" elif start_page: pages = start_page doaj_article_id = obj["id"].lower() ext_ids = self.doaj_ext_ids(bibjson["identifier"], doaj_article_id) abstracts = self.doaj_abstracts(bibjson) or [] contribs = self.doaj_contribs(bibjson.get("author") or []) or [] # DOAJ-specific extra doaj_extra: Dict[str, Any] = dict() if bibjson.get("subject"): doaj_extra["subject"] = bibjson.get("subject") if bibjson.get("keywords"): doaj_extra["keywords"] = [ k for k in [clean_str(s) for s in bibjson.get("keywords")] if k ] # generic extra extra: Dict[str, Any] = dict() if country: extra["country"] = country if not container_id and container_name: extra["container_name"] = container_name if release_year and release_month: # TODO: schema migration extra["release_month"] = release_month if doaj_extra: extra["doaj"] = doaj_extra re = fatcat_openapi_client.ReleaseEntity( work_id=None, container_id=container_id, release_type="article-journal", release_stage="published", title=title, release_year=release_year, # release_date, publisher=publisher, ext_ids=ext_ids, contribs=contribs or None, volume=volume, issue=issue, pages=pages, language=language, abstracts=abstracts or None, extra=extra or None, license_slug=license_slug, ) re = self.biblio_hacks(re) # TODO: filter out some of these by publishers which are known to # register DOIs. eg, PLOS, maybe others return re
def parse_record(self, xml_elem): """ - title => may contain <i>, <sub>, <sup>, <tt> - journal (abbrev?) - volume, pages, number (number -> issue) - publisher - year => for conferences, year of conference not of publication - month - crossref (from inproceedings to specific proceedings volume) - booktitle => for inproceedings, this is the name of conference or workshop. acronym. - isbn """ dblp_key = xml_elem.get('key') if not dblp_key: self.counts['skip-empty-key'] += 1 return False dblp_key_type = dblp_key.split('/')[0] # dblp_prefix may be used for container lookup dblp_prefix = None if dblp_key_type in ('journals', 'conf'): dblp_prefix = '/'.join(dblp_key.split('/')[:2]) elif dblp_key_type in ('series', 'reference', 'tr', 'books'): dblp_prefix = '/'.join(dblp_key.split('/')[:-1]) publtype = xml_elem.get('publtype') or None dblp_type = xml_elem.name if dblp_type not in self.ELEMENT_TYPES: self.counts[f'skip-dblp-type:{dblp_type}'] += 1 if dblp_key_type in ('homepages', 'persons', 'dblpnote'): self.counts['skip-key-type'] += 1 return False if dblp_key.startswith('journals/corr/'): self.counts['skip-arxiv-corr'] += 1 return False title = clean_str(" ".join(xml_elem.title.stripped_strings), force_xml=True) if not title: self.counts['skip-title'] += 1 return False if title.endswith('.'): title = title[:-1] release_type = None release_stage = 'published' withdrawn_status = None # primary releae_type detection: type of XML element, then prefix of key for granularity if dblp_type == 'article': release_type = 'article' if dblp_key_type == 'journals' and publtype != 'informal': release_type = 'article-journal' elif dblp_key_type == 'tr': release_type = 'report' elif title.startswith("Review:"): release_type = 'review' elif dblp_type == 'inproceedings': release_type = 'paper-conference' elif dblp_type == 'book': release_type = 'book' elif dblp_type == 'incollection': # XXX: part vs. chapter? release_type = 'chapter' elif dblp_type == 'data': release_type = 'dataset' elif dblp_type in ('mastersthesis', 'phdthesis'): release_type = 'thesis' # overrides/extensions of the above if publtype == 'informal': # for conferences, seems to indicate peer-review status # for journals, seems to indicate things like book reviews; split out above pass elif publtype == 'encyclopedia': release_type = 'entry-encyclopedia' elif publtype == 'edited': # XXX: article? release_type = 'editorial' elif publtype == 'data': release_type = 'dataset' elif publtype == 'data': release_type = 'dataset' elif publtype == 'software': release_type = 'software' elif publtype == 'widthdrawn': withdrawn_status = 'widthdrawn' elif publtype == 'survey': # XXX: flag as a review/survey article? pass #print((release_type, dblp_type, dblp_key_type, publtype), file=sys.stderr) container_name = None booktitle = clean_str(xml_elem.booktitle and xml_elem.booktitle.text) series = clean_str(xml_elem.series and xml_elem.series.text) if xml_elem.journal: container_name = clean_str(xml_elem.journal.text) container_id = None if dblp_prefix: container_id = self.lookup_dblp_prefix(dblp_prefix) # note: we will skip later if couldn't find prefix publisher = clean_str(xml_elem.publisher and xml_elem.publisher.text) volume = clean_str(xml_elem.volume and xml_elem.volume.text) issue = clean_str(xml_elem.number and xml_elem.number.text) pages = clean_str(xml_elem.pages and xml_elem.pages.text) release_year = clean_str(xml_elem.year and xml_elem.year.text) if release_year and release_year.isdigit(): release_year = int(release_year) else: release_year = None release_month = parse_month(clean_str(xml_elem.month and xml_elem.month.text)) isbn = clean_isbn13(xml_elem.isbn and xml_elem.isbn.text) part_of_key = clean_str(xml_elem.crossref and xml_elem.crossref.text) # block bogus far-future years/dates if release_year is not None and (release_year > (self.this_year + 5) or release_year < 1000): release_month = None release_year = None contribs = self.dblp_contribs(xml_elem or []) ext_ids = self.dblp_ext_ids(xml_elem, dblp_key) if isbn: ext_ids.isbn13 = isbn if ext_ids.doi: self.counts['has-doi'] += 1 # dblp-specific extra dblp_extra = dict(type=dblp_type) note = clean_str(xml_elem.note and xml_elem.note.text) if note and not 'base-search.net' in note: dblp_extra['note'] = note if part_of_key: dblp_extra['part_of_key'] = part_of_key # generic extra extra = dict() if not container_id and container_name: extra['container_name'] = container_name if series and (dblp_key_type == 'series' or dblp_type == 'book'): extra['series-title'] = series elif series: dblp_extra['series'] = series if booktitle and dblp_key_type == 'series': extra['container-title'] = booktitle elif booktitle and dblp_key_type == 'conf': extra['event'] = booktitle elif booktitle: dblp_extra['booktitle'] = booktitle if release_year and release_month: # TODO: release_month schema migration extra['release_month'] = release_month if dblp_extra: extra['dblp'] = dblp_extra if not extra: extra = None re = fatcat_openapi_client.ReleaseEntity( work_id=None, container_id=container_id, release_type=release_type, release_stage=release_stage, withdrawn_status=withdrawn_status, title=title, release_year=release_year, #release_date, publisher=publisher, ext_ids=ext_ids, contribs=contribs, volume=volume, issue=issue, pages=pages, extra=extra, ) re = self.biblio_hacks(re) if self.dump_json_mode: re_dict = entity_to_dict(re, api_client=self.api.api_client) re_dict['_dblp_ee_urls'] = self.dblp_ext_urls(xml_elem) re_dict['_dblp_prefix'] = dblp_prefix print(json.dumps(re_dict, sort_keys=True)) return False if not re.container_id: self.counts["skip-dblp-container-missing"] += 1 return False return re
def parse_record(self, obj): """ bibjson { abstract (string, optional), author (Array[bibjson.author], optional), identifier (Array[bibjson.identifier]), journal (bibjson.journal, optional), keywords (Array[string], optional), link (Array[bibjson.link], optional), month (string, optional), subject (Array[bibjson.subject], optional), title (string), year (string, optional) } bibjson.journal { country (string, optional), end_page (string, optional), language (Array[string], optional), license (Array[bibjson.journal.license], optional), number (string, optional), publisher (string, optional), start_page (string, optional), title (string, optional), volume (string, optional) } """ if not obj or not isinstance(obj, dict) or not 'bibjson' in obj: self.counts['skip-empty'] += 1 return None bibjson = obj['bibjson'] title = clean_str(bibjson.get('title'), force_xml=True) if not title: self.counts['skip-title'] += 1 return False container_name = clean_str(bibjson['journal']['title']) container_id = None # NOTE: 'issns' not documented in API schema for issn in bibjson['journal']['issns']: issnl = self.issn2issnl(issn) if issnl: container_id = self.lookup_issnl(self.issn2issnl(issn)) if container_id: # don't store container_name when we have an exact match container_name = None break volume = clean_str(bibjson['journal'].get('volume')) # NOTE: this schema seems to use "number" as "issue number" issue = clean_str(bibjson['journal'].get('number')) publisher = clean_str(bibjson['journal'].get('publisher')) try: release_year = int(bibjson.get('year')) except (TypeError, ValueError): release_year = None release_month = parse_month(clean_str(bibjson.get('month'))) # block bogus far-future years/dates if release_year is not None and (release_year > (self.this_year + 5) or release_year < 1000): release_month = None release_year = None license_slug = self.doaj_license_slug( bibjson['journal'].get('license')) country = parse_country_name(bibjson['journal'].get('country')) language = None for raw in bibjson['journal'].get('language') or []: language = parse_lang_name(raw) if language: break # pages # NOTE: error in API docs? seems like start_page not under 'journal' object start_page = clean_str( bibjson['journal'].get('start_page')) or clean_str( bibjson.get('start_page')) end_page = clean_str(bibjson['journal'].get('end_page')) or clean_str( bibjson.get('end_page')) pages: Optional[str] = None if start_page and end_page: pages = f"{start_page}-{end_page}" elif start_page: pages = start_page doaj_article_id = obj['id'].lower() ext_ids = self.doaj_ext_ids(bibjson['identifier'], doaj_article_id) abstracts = self.doaj_abstracts(bibjson) contribs = self.doaj_contribs(bibjson.get('author') or []) # DOAJ-specific extra doaj_extra = dict() if bibjson.get('subject'): doaj_extra['subject'] = bibjson.get('subject') if bibjson.get('keywords'): doaj_extra['keywords'] = [ k for k in [clean_str(s) for s in bibjson.get('keywords')] if k ] # generic extra extra = dict() if country: extra['country'] = country if not container_id and container_name: extra['container_name'] = container_name if release_year and release_month: # TODO: schema migration extra['release_month'] = release_month if doaj_extra: extra['doaj'] = doaj_extra if not extra: extra = None re = fatcat_openapi_client.ReleaseEntity( work_id=None, container_id=container_id, release_type='article-journal', release_stage='published', title=title, release_year=release_year, #release_date, publisher=publisher, ext_ids=ext_ids, contribs=contribs, volume=volume, issue=issue, pages=pages, language=language, abstracts=abstracts, extra=extra, license_slug=license_slug, ) re = self.biblio_hacks(re) return re
def parse_record(self, a: Any) -> ReleaseEntity: medline = a.MedlineCitation # PubmedData isn't required by DTD, but seems to always be present pubmed = a.PubmedData extra = dict() extra_pubmed = dict() identifiers = pubmed.ArticleIdList pmid = medline.PMID.string.strip() doi = identifiers.find("ArticleId", IdType="doi") if doi and doi.string: doi = clean_doi(doi.string) else: doi = None pmcid = identifiers.find("ArticleId", IdType="pmc") if pmcid: pmcid = clean_pmcid(pmcid.string.strip().upper()) release_type = None pub_types = [] for pub_type in medline.Article.PublicationTypeList.find_all("PublicationType"): pub_types.append(pub_type.string) if pub_type.string in PUBMED_RELEASE_TYPE_MAP: release_type = PUBMED_RELEASE_TYPE_MAP[pub_type.string] break if pub_types: extra_pubmed["pub_types"] = pub_types if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): release_type = "retraction" retraction_of = medline.find("CommentsCorrections", RefType="RetractionOf") if retraction_of: if retraction_of.RefSource: extra_pubmed["retraction_of_raw"] = retraction_of.RefSource.string if retraction_of.PMID: extra_pubmed["retraction_of_pmid"] = retraction_of.PMID.string # everything in medline is published release_stage = "published" if medline.Article.PublicationTypeList.find(string="Corrected and Republished Article"): release_stage = "updated" if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): release_stage = "retraction" withdrawn_status = None if medline.Article.PublicationTypeList.find(string="Retracted Publication"): withdrawn_status = "retracted" elif medline.find("CommentsCorrections", RefType="ExpressionOfConcernIn"): withdrawn_status = "concern" pages = medline.find("MedlinePgn") if pages: pages = pages.string title = medline.Article.ArticleTitle.get_text() # always present if title: title = title.replace("\n", " ") if title.endswith("."): title = title[:-1] # this hides some "special" titles, but the vast majority are # translations; translations don't always include the original_title if title.startswith("[") and title.endswith("]"): title = title[1:-1] else: # will filter out later title = None original_title = medline.Article.find("VernacularTitle", recurse=False) if original_title: original_title = original_title.get_text() or None original_title = original_title.replace("\n", " ") if original_title and original_title.endswith("."): original_title = original_title[:-1] if original_title and not title: # if we only have an "original" title, but not translated/english # title, sub in the original title so the entity can be created title = original_title original_title = None # TODO: happening in alpha order, not handling multi-language well. language = medline.Article.Language if language: language = language.get_text() if language in ("und", "un"): # "undetermined" language = None else: language = LANG_MAP_MARC.get(language) if not language and not (medline.Article.Language.get_text() in LANG_MAP_MARC): warnings.warn( "MISSING MARC LANG: {}".format(medline.Article.Language.string) ) ### Journal/Issue Metadata # MedlineJournalInfo is always present issnl = None container_id = None container_name = None container_extra = dict() mji = medline.MedlineJournalInfo if mji.find("Country"): country_name = mji.Country.string.strip() country_code = COUNTRY_NAME_MAP.get(country_name) if country_code: container_extra["country"] = country_code elif country_name: container_extra["country_name"] = country_name if mji.find("ISSNLinking"): issnl = mji.ISSNLinking.string journal = medline.Article.Journal issnp = journal.find("ISSN", IssnType="Print") if issnp: issnp = clean_issn(issnp.string) else: issnp = None if not issnl and issnp: issnl = self.issn2issnl(issnp) else: issnl = None if issnl: container_id = self.lookup_issnl(issnl) pub_date = medline.Article.find("ArticleDate") if not pub_date: pub_date = journal.PubDate if not pub_date: pub_date = journal.JournalIssue.PubDate release_date: Optional[str] = None release_year: Optional[int] = None if pub_date.Year: release_year = int(pub_date.Year.string) if pub_date.find("Day") and pub_date.find("Month"): try: release_date_date = datetime.date( release_year, MONTH_ABBR_MAP[pub_date.Month.string], int(pub_date.Day.string), ) release_date = release_date_date.isoformat() except ValueError as ve: print("bad date, skipping: {}".format(ve), file=sys.stderr) release_date = None elif pub_date.MedlineDate: medline_date = pub_date.MedlineDate.string.strip() if len(medline_date) >= 4 and medline_date[:4].isdigit(): release_year = int(medline_date[:4]) if release_year < 1300 or release_year > 2040: print( "bad medline year, skipping: {}".format(release_year), file=sys.stderr ) release_year = None else: print( "unparsable medline date, skipping: {}".format(medline_date), file=sys.stderr, ) if journal.find("Title"): container_name = journal.Title.get_text() if ( container_id is None and self.create_containers and (issnl is not None) and container_name ): # name, type, publisher, issnl # extra: original_name, languages, country ce = fatcat_openapi_client.ContainerEntity( name=container_name, container_type="journal", # NOTE: publisher not included issnl=issnl, issnp=issnp, extra=(container_extra or None), ) ce_edit = self.create_container(ce) container_id = ce_edit.ident self._issnl_id_map[issnl] = container_id ji = journal.JournalIssue volume = None if ji.find("Volume"): volume = ji.Volume.string issue = None if ji.find("Issue"): issue = ji.Issue.string ### Abstracts # "All abstracts are in English" abstracts = [] primary_abstract = medline.find("Abstract") if primary_abstract and primary_abstract.AbstractText.get("NlmCategory"): joined = "\n".join( [m.get_text() for m in primary_abstract.find_all("AbstractText")] ) abst = fatcat_openapi_client.ReleaseAbstract( content=joined, mimetype="text/plain", lang="en", ) if abst.content: abstracts.append(abst) elif primary_abstract: for abstract in primary_abstract.find_all("AbstractText"): abst = fatcat_openapi_client.ReleaseAbstract( content=abstract.get_text().strip(), mimetype="text/plain", lang="en", ) if abst.content: abstracts.append(abst) if abstract.find("math"): abst = fatcat_openapi_client.ReleaseAbstract( # strip the <AbstractText> tags content=str(abstract)[14:-15], mimetype="application/mathml+xml", lang="en", ) if abst.content: abstracts.append(abst) other_abstracts = medline.find_all("OtherAbstract") for other in other_abstracts: lang: Optional[str] = "en" if other.get("Language"): lang = LANG_MAP_MARC.get(other["Language"]) abst = fatcat_openapi_client.ReleaseAbstract( content=other.AbstractText.get_text().strip(), mimetype="text/plain", lang=lang, ) if abst.content: abstracts.append(abst) ### Contribs contribs = [] if medline.AuthorList: for author in medline.AuthorList.find_all("Author"): creator_id = None given_name = None surname = None raw_name = None if author.ForeName: given_name = author.ForeName.get_text().replace("\n", " ") if author.LastName: surname = author.LastName.get_text().replace("\n", " ") if given_name and surname: raw_name = "{} {}".format(given_name, surname) elif surname: raw_name = surname if not raw_name and author.CollectiveName and author.CollectiveName.get_text(): raw_name = author.CollectiveName.get_text().replace("\n", " ") contrib_extra = dict() orcid = author.find("Identifier", Source="ORCID") if orcid: # needs re-formatting from, eg, "0000000179841889" orcid = orcid.string if orcid.startswith("http://orcid.org/"): orcid = orcid.replace("http://orcid.org/", "") elif orcid.startswith("https://orcid.org/"): orcid = orcid.replace("https://orcid.org/", "") elif "-" not in orcid: orcid = "{}-{}-{}-{}".format( orcid[0:4], orcid[4:8], orcid[8:12], orcid[12:16], ) creator_id = self.lookup_orcid(orcid) contrib_extra["orcid"] = orcid affiliations = author.find_all("Affiliation") raw_affiliation = None if affiliations: raw_affiliation = affiliations[0].get_text().replace("\n", " ") if len(affiliations) > 1: contrib_extra["more_affiliations"] = [ ra.get_text().replace("\n", " ") for ra in affiliations[1:] ] if author.find("EqualContrib"): # TODO: schema for this? contrib_extra["equal"] = True contribs.append( fatcat_openapi_client.ReleaseContrib( raw_name=raw_name, given_name=given_name, surname=surname, role="author", raw_affiliation=raw_affiliation, creator_id=creator_id, extra=contrib_extra, ) ) if medline.AuthorList["CompleteYN"] == "N": contribs.append(fatcat_openapi_client.ReleaseContrib(raw_name="et al.")) for i, contrib in enumerate(contribs): if contrib.raw_name != "et al.": contrib.index = i ### References refs = [] if pubmed.ReferenceList: # note that Reference always exists within a ReferenceList, but # that there may be multiple ReferenceList (eg, sometimes one per # Reference) for ref in pubmed.find_all("Reference"): ref_extra: Dict[str, Any] = dict() ref_doi = ref.find("ArticleId", IdType="doi") if ref_doi: ref_doi = clean_doi(ref_doi.string) ref_pmid = ref.find("ArticleId", IdType="pubmed") if ref_pmid: ref_pmid = clean_pmid(ref_pmid.string) ref_release_id = None if ref_doi: ref_extra["doi"] = ref_doi if self.lookup_refs: ref_release_id = self.lookup_doi(ref_doi) if ref_pmid: ref_extra["pmid"] = ref_pmid if self.lookup_refs: ref_release_id = self.lookup_pmid(ref_pmid) ref_raw = ref.Citation if ref_raw: ref_extra["unstructured"] = ref_raw.get_text() refs.append( fatcat_openapi_client.ReleaseRef( target_release_id=ref_release_id, extra=ref_extra or None, ) ) # extra: # translation_of # aliases # container_name # group-title # pubmed: retraction refs if extra_pubmed: extra["pubmed"] = extra_pubmed title = clean_str(title) if not title: return None re = fatcat_openapi_client.ReleaseEntity( work_id=None, title=title, original_title=clean_str(original_title), release_type=release_type, release_stage=release_stage, release_date=release_date, release_year=release_year, withdrawn_status=withdrawn_status, ext_ids=fatcat_openapi_client.ReleaseExtIds( doi=doi, pmid=pmid, pmcid=pmcid, # isbn13 # never in Article ), volume=volume, issue=issue, pages=pages, # publisher # not included? language=language, # license_slug # not in MEDLINE abstracts=abstracts or None, contribs=contribs or None, refs=refs or None, container_id=container_id, extra=extra or None, ) return re
def parse_record(self, xml_elem: Any) -> Optional[ReleaseEntity]: """ - title => may contain <i>, <sub>, <sup>, <tt> - journal (abbrev?) - volume, pages, number (number -> issue) - publisher - year => for conferences, year of conference not of publication - month - crossref (from inproceedings to specific proceedings volume) - booktitle => for inproceedings, this is the name of conference or workshop. acronym. - isbn """ dblp_key = xml_elem.get("key") if not dblp_key: self.counts["skip-empty-key"] += 1 return False dblp_key_type = dblp_key.split("/")[0] # dblp_prefix may be used for container lookup dblp_prefix = None if dblp_key_type in ("journals", "conf"): dblp_prefix = "/".join(dblp_key.split("/")[:2]) elif dblp_key_type in ("series", "reference", "tr", "books"): dblp_prefix = "/".join(dblp_key.split("/")[:-1]) publtype = xml_elem.get("publtype") or None dblp_type = xml_elem.name if dblp_type not in self.ELEMENT_TYPES: self.counts[f"skip-dblp-type:{dblp_type}"] += 1 if dblp_key_type in ("homepages", "persons", "dblpnote"): self.counts["skip-key-type"] += 1 return False if dblp_key.startswith("journals/corr/"): self.counts["skip-arxiv-corr"] += 1 return False title = clean_str(" ".join(xml_elem.title.stripped_strings), force_xml=True) if not title: self.counts["skip-title"] += 1 return False if title.endswith("."): title = title[:-1] release_type = None release_stage = "published" withdrawn_status = None # primary releae_type detection: type of XML element, then prefix of key for granularity if dblp_type == "article": release_type = "article" if dblp_key_type == "journals" and publtype != "informal": release_type = "article-journal" elif dblp_key_type == "tr": release_type = "report" elif title.startswith("Review:"): release_type = "review" elif dblp_type == "inproceedings": release_type = "paper-conference" elif dblp_type == "book": release_type = "book" elif dblp_type == "incollection": # XXX: part vs. chapter? release_type = "chapter" elif dblp_type == "data": release_type = "dataset" elif dblp_type in ("mastersthesis", "phdthesis"): release_type = "thesis" # overrides/extensions of the above if publtype == "informal": # for conferences, seems to indicate peer-review status # for journals, seems to indicate things like book reviews; split out above pass elif publtype == "encyclopedia": release_type = "entry-encyclopedia" elif publtype == "edited": # XXX: article? release_type = "editorial" elif publtype == "data": release_type = "dataset" elif publtype == "data": release_type = "dataset" elif publtype == "software": release_type = "software" elif publtype == "widthdrawn": withdrawn_status = "widthdrawn" elif publtype == "survey": # XXX: flag as a review/survey article? pass # print((release_type, dblp_type, dblp_key_type, publtype), file=sys.stderr) container_name = None booktitle = clean_str(xml_elem.booktitle and xml_elem.booktitle.text) series = clean_str(xml_elem.series and xml_elem.series.text) if xml_elem.journal: container_name = clean_str(xml_elem.journal.text) container_id = None if dblp_prefix: container_id = self.lookup_dblp_prefix(dblp_prefix) # note: we will skip later if couldn't find prefix publisher = clean_str(xml_elem.publisher and xml_elem.publisher.text) volume = clean_str(xml_elem.volume and xml_elem.volume.text) issue = clean_str(xml_elem.number and xml_elem.number.text) pages = clean_str(xml_elem.pages and xml_elem.pages.text) release_year_str = clean_str(xml_elem.year and xml_elem.year.text) if release_year_str and release_year_str.isdigit(): release_year: Optional[int] = int(release_year_str) else: release_year = None release_month = parse_month( clean_str(xml_elem.month and xml_elem.month.text)) isbn = clean_isbn13(xml_elem.isbn and xml_elem.isbn.text) part_of_key = clean_str(xml_elem.crossref and xml_elem.crossref.text) # block bogus far-future years/dates if release_year is not None and (release_year > (self.this_year + 5) or release_year < 1000): release_month = None release_year = None contribs = self.dblp_contribs(xml_elem) ext_ids = self.dblp_ext_ids(xml_elem, dblp_key) if isbn: ext_ids.isbn13 = isbn if ext_ids.doi: self.counts["has-doi"] += 1 # dblp-specific extra dblp_extra = dict(type=dblp_type) note = clean_str(xml_elem.note and xml_elem.note.text) if note and "base-search.net" not in note: dblp_extra["note"] = note if part_of_key: dblp_extra["part_of_key"] = part_of_key # generic extra extra: Dict[str, Any] = dict() if not container_id and container_name: extra["container_name"] = container_name if series and (dblp_key_type == "series" or dblp_type == "book"): extra["series-title"] = series elif series: dblp_extra["series"] = series if booktitle and dblp_key_type == "series": extra["container-title"] = booktitle elif booktitle and dblp_key_type == "conf": extra["event"] = booktitle elif booktitle: dblp_extra["booktitle"] = booktitle if release_year and release_month: # TODO: release_month schema migration extra["release_month"] = release_month if dblp_extra: extra["dblp"] = dblp_extra re = fatcat_openapi_client.ReleaseEntity( work_id=None, container_id=container_id, release_type=release_type, release_stage=release_stage, withdrawn_status=withdrawn_status, title=title, release_year=release_year, # release_date, publisher=publisher, ext_ids=ext_ids, contribs=contribs or None, volume=volume, issue=issue, pages=pages, extra=extra or None, ) re = self.biblio_hacks(re) if self.dump_json_mode: re_dict = entity_to_dict(re, api_client=self.api.api_client) re_dict["_dblp_ee_urls"] = self.dblp_ext_urls(xml_elem) re_dict["_dblp_prefix"] = dblp_prefix print(json.dumps(re_dict, sort_keys=True)) return False if not re.container_id: self.counts["skip-dblp-container-missing"] += 1 return False return re
def parse_record(self, obj: Dict[str, Any]) -> Optional[ReleaseEntity]: """ Mapping datacite JSON to ReleaseEntity. """ if not obj or not isinstance(obj, dict): return None if "attributes" not in obj: return None attributes = obj["attributes"] doi = clean_doi(attributes.get("doi", "").lower()) if not doi: print("skipping record without a DOI", file=sys.stderr) return None if not str.isascii(doi): print("[{}] skipping non-ascii doi for now".format(doi)) return None creators = attributes.get("creators", []) or [] contributors = attributes.get("contributors", []) or [ ] # Much fewer than creators. contribs = self.parse_datacite_creators(creators, doi=doi) # Beside creators, we have contributors in datacite. Sample: # ContactPerson, DataCollector, DataCurator, DataManager, Distributor, # Editor, Funder, HostingInstitution, Other, Producer, ProjectLeader, # ProjectMember, RelatedPerson, ResearchGroup, Researcher, # RightsHolder, Sponsor, Supervisor # # Datacite schema: # https://schema.datacite.org/meta/kernel-4.3/doc/DataCite-MetadataKernel_v4.3.pdf#page=32 # -- could be used as a form of controlled vocab? # # Currently (07/2020) in release_contrib: # # select count(*), role from release_contrib group by role; # count | role # -----------+------------ # 500269665 | author # 4386563 | editor # 17871 | translator # 10870584 | # (4 rows) # # Related: https://guide.fatcat.wiki/entity_release.html -- role # (string, of a set): the type of contribution, from a controlled # vocabulary. TODO: vocabulary needs review. contribs_extra_contributors = self.parse_datacite_creators( contributors, set_index=False, doi=doi) # Unfortunately, creators and contributors might overlap, refs GH59. for cc in contribs_extra_contributors: if contributor_list_contains_contributor(contribs, cc): continue contribs.append(cc) # Title, may come with "attributes.titles[].titleType", like # "AlternativeTitle", "Other", "Subtitle", "TranslatedTitle" titles = attributes.get("titles", []) or [] title, original_language_title, subtitle = parse_datacite_titles( titles) if title is None: print("[{}] skipping record w/o title: {}".format(doi, obj), file=sys.stderr) return False title = clean_str(title) if not title: print("[{}] skipping record w/o title: {}".format(doi, obj), file=sys.stderr) return False # check for blocklisted "spam", e.g. "FULL MOVIE" for rule in DATACITE_TITLE_SPAM_WORDGROUPS: seen = set() token_list: List[str] = rule.get("tokens") or [] for token in token_list: if token in title.lower(): seen.add(token) if len(seen) >= rule["min"]: print("[{}] skipping spammy title: {}".format(doi, obj), file=sys.stderr) return False if not subtitle: subtitle = None else: subtitle = clean_str(subtitle) # Dates. A few internal dates (registered, created, updated) and # published (0..2554). We try to work with typed date list, in # "attributes.dates[].dateType", values: "Accepted", "Available" # "Collected", "Copyrighted", "Created", "Issued", "Submitted", # "Updated", "Valid". release_date, release_month, release_year = parse_datacite_dates( attributes.get("dates", [])) # block bogus far-future years/dates if release_year is not None and (release_year > (self.this_year + 5) or release_year < 1000): release_date = None release_month = None release_year = None # Some records do not use the "dates" field (e.g. micropub), but: # "attributes.published" or "attributes.publicationYear" if not any((release_date, release_month, release_year)): release_date, release_month, release_year = parse_single_date( attributes.get("publicationYear")) if not any((release_date, release_month, release_year)): release_date, release_month, release_year = parse_single_date( attributes.get("published")) if not any((release_date, release_month, release_year)): print("[{}] record w/o date: {}".format(doi, obj), file=sys.stderr) # Start with clear stages, e.g. published. TODO(martin): we could # probably infer a bit more from the relations, e.g. # "IsPreviousVersionOf" or "IsNewVersionOf". release_stage: Optional[str] = "published" # TODO(martin): If 'state' is not 'findable' or 'isActive' is not true, # we might want something else than 'published'. See also: # https://support.datacite.org/docs/doi-states. # Publisher. A few NA values. A few bogus values. publisher = attributes.get("publisher") if publisher in UNKNOWN_MARKERS | set(("Unpublished", "Unknown")): publisher = None release_stage = None if publisher is not None and len(publisher) > 80: # Arbitrary magic value max length. TODO(martin): better heuristic, # but factored out; first we have to log misses. Example: # "ETH-Bibliothek Zürich, Bildarchiv / Fotograf: Feller, # Elisabeth, Empfänger, Unbekannt, Fotograf / Fel_041033-RE / # Unbekannt, Nutzungsrechte müssen durch den Nutzer abgeklärt # werden" publisher = None if publisher: publisher = clean_str(publisher) # Container. For the moment, only ISSN as container. container_id = None container_name = None container = attributes.get("container", {}) or {} if container.get("type") in DATACITE_CONTAINER_TYPE_MAP.keys(): container_type = DATACITE_CONTAINER_TYPE_MAP.get(container["type"]) if container.get("identifier") and container.get( "identifierType") == "ISSN": issn = container.get("identifier") if issn and len(issn) == 8: issn = issn[:4] + "-" + issn[4:] issnl = self.issn2issnl(issn) else: issnl = None if issnl is not None: container_id = self.lookup_issnl(issnl) if container_id is None and container.get("title"): container_name = container.get("title") if isinstance(container_name, list): if len(container_name) > 0: print("[{}] too many container titles: {}". format(doi, len(container_name))) container_name = container_name[0] assert isinstance(container_name, str) ce = fatcat_openapi_client.ContainerEntity( issnl=issnl, container_type=container_type, name=container_name, ) ce_edit = self.create_container(ce) container_id = ce_edit.ident self._issnl_id_map[issnl] = container_id else: # TODO(martin): factor this out into a testable function. # TODO(martin): "container_name": "№1(1) (2018)" / 10.26087/inasan.2018.1.1.013 container_name = container.get("title") if isinstance(container_name, list): if len(container_name) > 0: print("[{}] too many container titles: {}".format( doi, len(container_name))) container_name = container_name[0] # Exception: https://www.micropublication.org/, see: !MR24. if container_id is None and container_name is None: if publisher and publisher.lower().startswith("micropublication"): container_name = publisher # Volume and issue. volume = container.get("volume") issue = container.get("issue") if volume: volume = clean_str(volume) if issue: issue = clean_str(issue) # Pages. pages = None first_page = container.get("firstPage") last_page = container.get("lastPage") if first_page and last_page: try: _ = int(first_page) < int(last_page) pages = "{}-{}".format(first_page, last_page) except ValueError as err: # noqa: F841 # TODO(martin): This is more debug than info. # print('[{}] {}'.format(doi, err), file=sys.stderr) pass if not pages and first_page: pages = first_page # License. license_slug = None license_extra = [] for lic in attributes.get("rightsList", []): slug = datacite_lookup_license_slug(lic.get("rightsUri")) if slug: license_slug = slug license_extra.append(lic) release_type = self.datacite_release_type(doi, attributes) # Language values are varied ("ger", "es", "English", "ENG", "en-us", # "other", ...). Try to crush it with langcodes: "It may sound to you # like langcodes solves a pretty boring problem. At one level, that's # right. Sometimes you have a boring problem, and it's great when a # library solves it for you." -- TODO(martin): We need more of these. language = None value = attributes.get("language", "") or "" try: language = pycountry.languages.lookup(value).alpha_2 except (LookupError, AttributeError) as err: # noqa: F841 pass # TODO(martin): Print this on debug level, only. # print('[{}] language lookup miss for {}: {}'.format(doi, value, err), file=sys.stderr) # Abstracts appear in "attributes.descriptions[].descriptionType", some # of the observed values: "Methods", "TechnicalInfo", # "SeriesInformation", "Other", "TableOfContents", "Abstract". The # "Other" fields might contain references or related articles (with # DOI). TODO(martin): maybe try to parse out some of those refs. abstracts = [] descs = attributes.get("descriptions", []) or [] for desc in descs: if not desc.get("descriptionType") == "Abstract": continue # Description maybe a string, int or list. text = desc.get("description", "") if not text: continue if isinstance(text, int): text = "{}".format(text) if isinstance(text, list): try: text = "\n".join(text) except TypeError: continue # Bail out, if it is not a list of strings. # Limit length. if len(text) < 10: continue if len(text) > MAX_ABSTRACT_LENGTH: text = text[:MAX_ABSTRACT_LENGTH] + " [...]" # Detect language. This is fuzzy and may be removed, if too unreliable. lang = None try: lang = langdetect.detect(text) except (langdetect.lang_detect_exception.LangDetectException, TypeError) as err: print( "[{}] language detection failed with {} on {}".format( doi, err, text), file=sys.stderr, ) abstract_text = clean_str(text) if not abstract_text: continue abstracts.append( fatcat_openapi_client.ReleaseAbstract( mimetype="text/plain", content=abstract_text, lang=lang, )) # References and relations. Datacite include many relation types in # "attributes.relatedIdentifiers[].relationType", e.g. # "IsPartOf", "IsPreviousVersionOf", "Continues", "IsVariantFormOf", # "IsSupplementTo", "Cites", "IsSupplementedBy", "IsDocumentedBy", "HasVersion", # "IsCitedBy", "IsMetadataFor", "IsNewVersionOf", "IsIdenticalTo", "HasPart", # "References", "Reviews", "HasMetadata", "IsContinuedBy", "IsVersionOf", # "IsDerivedFrom", "IsSourceOf". # # For the moment, we only care about References. refs, ref_index = [], 0 relIds = attributes.get("relatedIdentifiers", []) or [] for rel in relIds: if not rel.get("relationType", "") in ("References", "Cites"): continue ref_extra = dict() if rel.get("relatedIdentifierType", "") == "DOI": ref_extra["doi"] = rel.get("relatedIdentifier") refs.append( fatcat_openapi_client.ReleaseRef( index=ref_index, extra=ref_extra or None, )) ref_index += 1 # More specific release_type via 'Reviews' relationsship. for rel in relIds: if rel.get("relatedIdentifierType", "") != "Reviews": continue release_type = "review" # Extra information. extra_datacite: Dict[str, Any] = dict() if license_extra: extra_datacite["license"] = license_extra if attributes.get("subjects"): # these subjects with schemeUri are too much metadata, which # doesn't compress. filter them out. extra_subjects = [ subj for subj in attributes["subjects"] if not subj.get("schemeUri") ] if extra_subjects: extra_datacite["subjects"] = extra_subjects # Include version information. metadata_version = attributes.get("metadataVersion") or "" if metadata_version: extra_datacite["metadataVersion"] = metadata_version # Include resource types. types = attributes.get("types") or {} resource_type = types.get("resourceType", "") or "" resource_type_general = types.get("resourceTypeGeneral", "") or "" if resource_type and resource_type.lower( ) not in UNKNOWN_MARKERS_LOWER: extra_datacite["resourceType"] = resource_type if resource_type_general and resource_type_general.lower( ) not in UNKNOWN_MARKERS_LOWER: extra_datacite["resourceTypeGeneral"] = resource_type_general # Include certain relations from relatedIdentifiers. Keeping the # original structure of data here, which is a list of dicts, with # relation type, identifier and identifier type (mostly). relations = [] for rel in relIds: if rel.get("relationType") in ( "IsPartOf", "Reviews", "Continues", "IsVariantFormOf", "IsSupplementTo", "HasVersion", "IsMetadataFor", "IsNewVersionOf", "IsIdenticalTo", "IsVersionOf", "IsDerivedFrom", "IsSourceOf", ): relations.append(rel) # TODO: could use many of these relations to do release/work grouping if relations: extra_datacite["relations"] = relations extra: Dict[str, Any] = dict() # "1.0.0", "v1.305.2019", "Final", "v1.0.0", "v0.3.0", "1", "0.19.0", # "3.1", "v1.1", "{version}", "4.0", "10329", "11672", "11555", # "v1.4.5", "2", "V1", "v3.0", "v0", "v0.6", "11124", "v1.0-beta", "1st # Edition", "20191024", "v2.0.0", "v0.9.3", "10149", "2.0", null, # "v0.1.1", "3.0", "1.0", "3", "v1.12.2", "20191018", "v0.3.1", "v1.0", # "10161", "10010691", "10780", # "Presentación" version = attributes.get("version") or None # top-level extra keys if not container_id and container_name: extra["container_name"] = container_name # Always include datacite key, even if value is empty (dict). extra["datacite"] = extra_datacite # Preparation for a schema update. if release_month: extra["release_month"] = release_month # Assemble release. re = fatcat_openapi_client.ReleaseEntity( work_id=None, container_id=container_id, release_type=release_type, release_stage=release_stage, title=title, subtitle=subtitle, original_title=original_language_title, release_year=release_year, release_date=release_date, publisher=publisher, ext_ids=fatcat_openapi_client.ReleaseExtIds(doi=doi, ), contribs=contribs, volume=volume, issue=issue, pages=pages, language=language, abstracts=abstracts, refs=refs, extra=extra, license_slug=license_slug, version=version, ) re = self.biblio_hacks(re) return re