def get_journal_list(self):
journal_list = self.__file_university['journal']
journal_list_to_return = []
index = 0
for line in journal_list:
article_text = self.__file_university.iloc[index]
try:
temp_journal = models.Journal(line['title_full'],
line['publisher'], line['issn'],
line['country'], {})
except Exception as e:
index += 1
continue
temp_var = False
temp_article = self.create_article(article_text)
for compare_journal in journal_list_to_return:
if compare_journal.get_title() == temp_journal.get_title():
compare_journal.set_article(temp_article.get_volume(),
temp_article)
temp_var = True
break
if not temp_var:
temp_journal.set_article(temp_article.get_volume(),
temp_article)
journal_list_to_return.append(temp_journal)
index += 1
return journal_list_to_return
def get_journal(self):
g = Graph()
name = None
for jrnl in client.RelatedObject('Service', self.cid, 'SERV_has_JOUR'):
juri = models.journal_uri(jrnl.cid)
g.add((self.uri, VIVO.relates, juri))
jm = models.Journal(**jrnl.__dict__)
name = jrnl.name
g += jm.to_rdf()
return name, g
def create_journal(code):
j = models.Journal(code=code)
j.save()
return j
def insert_articles_into_db(collection_of_articles):
"""
Insert a dictionary of publications into separate database rows
from a BibTex file.
"""
paper_search_fields = ["title", "author", "year", "month", "volume",\
"number", "pages", "abstract", "doi", "keywords",]
# Collection of all the models that
# will be written in bulk
paper_models = []
author_models = []
journal_models = []
journal_list = []
for paper_item in collection_of_articles:
paper_record = {}
author_record = {}
keys_in_paper_item = paper_item.keys()
for search_item in paper_search_fields:
# There are some fields that are ambiguous.
# These are dealt later.
if not (search_item=="journal" or \
search_item=="booktitle"
):
if search_item in keys_in_paper_item:
paper_record[search_item] = paper_item[search_item]
else:
paper_record[search_item] = ""
if "journal" in keys_in_paper_item:
paper_record["journal"] = paper_item["journal"]
if "booktitle" in keys_in_paper_item:
paper_record["journal"] = paper_item["booktitle"]
paper_record["authors"] = paper_item["author"]
list_of_authors = paper_item["author"].split(" and ")
for author_entry in list_of_authors:
author_entry = author_entry.strip()
author_record["full_name"] = author_entry
# The model is created but not witten to the database.
author_model_item = models.Author(
full_name=author_record["full_name"])
author_models.append(author_model_item)
if paper_record["journal"] not in journal_list:
journal_model_item = models.Journal(name=paper_record["journal"], )
journal_models.append(journal_model_item)
journal_list.append(paper_record["journal"])
paper_model_item = models.Paper(paper_title=paper_record["title"], \
paper_year=paper_record["year"], \
paper_volume=paper_record["volume"], \
paper_issue=paper_record["number"], \
paper_pages=paper_record["pages"], \
paper_month=paper_record["month"], \
paper_doi=paper_record["doi"], \
paper_abstract=paper_record["abstract"], \
paper_keywords=paper_record["keywords"], \
paper_authors=paper_record["authors"], \
paper_journal=paper_record["journal"]
)
paper_models.append(paper_model_item)
# The models are written in bulk
models.Paper.objects.bulk_create(paper_models)
models.Journal.objects.bulk_create(journal_models)
models.Author.objects.bulk_create(author_models)
return
def insert_articles_from_web(collection_of_articles):
"""
Insert a dictionary of publications into separate database rows
from the IEEE gateway.
"""
paper_search_fields = ["title", "authors", "affiliations", "pubtitle", "punumber", \
"pubtype", "publisher", "volume", "issue", "py", "spage", \
"epage", "abstract", "issn", "arnumber", "doi", "publicationId", \
"partnum", "mdurl", "pdf", "term", "month"]
journal_search_fields = ["pubtitle", "pubtype", "publisher", "issn"]
all_paper_list = []
all_author_list = []
all_journal_list = []
paper_models = []
author_models = []
journal_models = []
for paper_item in collection_of_articles:
paper_record = {}
journal_record = {}
author_record = {}
keys_in_paper_item = paper_item.keys()
for search_item in paper_search_fields:
# Some fields are different from the database.
# These are dealt with below.
if not (search_item=="spage" or \
search_item=="epage" or \
search_item=="term" or \
search_item=="publicationId" or \
search_item=="partnum" or \
search_item=="arnumber"
):
if search_item in keys_in_paper_item:
paper_record[search_item] = paper_item[search_item]
else:
paper_record[search_item] = ""
print(search_item, paper_record[search_item])
if "spage" in keys_in_paper_item:
paper_record["pages"] = paper_item["spage"] + "-"
else:
paper_record["pages"] = "-"
if "epage" in keys_in_paper_item:
paper_record["pages"] += paper_item["epage"]
if "term" in keys_in_paper_item:
paper_record["keywords"] = ""
for term_item in paper_item["term"]:
paper_record["keywords"] += term_item + ", "
paper_record["keywords"] = paper_record["keywords"][:-2]
else:
paper_record["keywords"] = ""
if "publicationId" in keys_in_paper_item:
paper_record["arnumber"] = paper_item["publicationId"]
elif "partnum" in keys_in_paper_item:
paper_record["arnumber"] = paper_item["partnum"]
elif "arnumber" in keys_in_paper_item:
paper_record["arnumber"] = paper_item["arnumber"]
for search_item in journal_search_fields:
journal_record[search_item] = paper_item[search_item]
# Authors are separated by ; but need to be separated by and
list_of_authors = paper_item["authors"].split(";")
list_of_authors_in_paper = ""
for count, author_entry in enumerate(list_of_authors):
author_entry = author_entry.strip()
author_record["full_name"] = author_entry
all_author_list.append(author_record)
author_model_item = models.Author(
full_name=author_record["full_name"])
author_models.append(author_model_item)
list_of_authors_in_paper += author_entry
if count < len(list_of_authors) - 1:
list_of_authors_in_paper += " and "
paper_record["authors"] = list_of_authors_in_paper
paper_model_item = models.Paper(paper_title=paper_record["title"], \
paper_year=paper_record["py"], \
paper_volume=paper_record["volume"], \
paper_issue=paper_record["issue"], \
paper_number=paper_record["punumber"], \
paper_pages=paper_record["pages"], \
paper_month=paper_record["month"], \
paper_doi=paper_record["doi"], \
paper_abstract=paper_record["abstract"], \
paper_keywords=paper_record["keywords"], \
paper_journal=paper_record["pubtitle"], \
paper_authors=paper_record["authors"], \
paper_arnumber=paper_record["arnumber"], \
paper_url=paper_record["mdurl"], \
paper_pdflink=paper_record["pdf"], \
publisher_organization=paper_record["publisher"], \
publisher_issn_number=paper_record["issn"], \
publisher_type=paper_record["pubtype"]
)
paper_models.append(paper_model_item)
all_paper_list.append(paper_record)
journal_model_item = models.Journal(name=paper_record["pubtitle"], \
organization=paper_record["publisher"], \
issn_number=paper_record["issn"], \
pub_type=paper_record["pubtype"]
)
journal_models.append(journal_model_item)
all_journal_list.append(journal_record)
# if "affiliations" in paper_item.keys():
# new_institution = models.Institution()
# new_institution.name = paper_item["affiliations"]
# new_institution.save()
#
# if list_of_authors_in_paper:
# for author_item in list_of_authors_in_paper:
# new_affiliation = models.Affiliation()
# new_affiliation.institution = new_institution
# new_affiliation.author = author_item
# new_affiliation.year = new_paper_entry.paper_year
# new_affiliation.save()
models.Paper.objects.bulk_create(paper_models)
models.Journal.objects.bulk_create(journal_models)
models.Author.objects.bulk_create(author_models)
return