def read_excel(fp):
"""Excel file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the Excel file (.xlsx).
Returns
-------
list:
List with entries.
"""
try:
dfs = pd.read_excel(fp, sheet_name=None)
except UnicodeDecodeError:
dfs = pd.read_excel(fp, sheet_name=None, encoding="ISO-8859-1")
best_sheet = None
sheet_obj_val = -1
wanted_columns = []
for type_name, type_list in COLUMN_DEFINITIONS.items():
wanted_columns.extend(type_list)
for sheet_name in dfs:
col_names = set([col.lower() for col in list(dfs[sheet_name])])
obj_val = len(col_names & set(wanted_columns))
if obj_val > sheet_obj_val:
sheet_obj_val = obj_val
best_sheet = sheet_name
return standardize_dataframe(dfs[best_sheet])
def read_csv(data_fp):
"""CVS file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the CSV file.
Returns
-------
list:
List with entries.
"""
for encoding in ["utf-8", "ISO-8859-1"]:
try:
df = pd.read_csv(data_fp,
sep=None,
encoding=encoding,
engine='python')
return standardize_dataframe(df)
except UnicodeDecodeError:
# if unicode error, go to next encoding
continue
raise UnicodeDecodeError("The encoding of the file is not supported.")
def read_pubmed_xml(fp):
"""PubMed XML file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the XML file (.xml).
Returns
-------
list:
List with entries.
"""
tree = ET.parse(fp)
root = tree.getroot()
records = []
for child in root:
parts = []
elem = child.find('MedlineCitation/Article/ArticleTitle')
title = elem.text.replace('[', '').replace(']', '')
for elem in child.iter('AbstractText'):
parts.append(elem.text)
authors = []
for author in child.iter('Author'):
author_elems = []
for elem in author.iter('ForeName'):
author_elems.append(elem.text)
for elem in author.iter('LastName'):
author_elems.append(elem.text)
authors.append(" ".join(author_elems))
author_str = ", ".join(authors)
abstract = " ".join(parts)
keyword_list = [keyword.text for keyword in child.iter('Keyword')]
keywords = ", ".join(keyword_list)
new_record = {
"abstract": abstract,
"title": title,
"authors": author_str,
"keywords": keywords,
}
records.append(new_record)
return standardize_dataframe(pd.DataFrame(records))
def read_ris(fp):
"""RIS file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the RIS file.
label: bool
Check for label. If None, this is automatic.
Returns
-------
list:
List with entries.
"""
encodings = ['ISO-8859-1', 'utf-8', 'utf-8-sig']
entries = None
for encoding in encodings:
try:
with open(fp, 'r', encoding=encoding) as bibliography_file:
mapping = _tag_key_mapping(reverse=False)
entries = list(readris(bibliography_file, mapping=mapping))
break
except UnicodeDecodeError:
pass
except IOError as e:
logging.warning(e)
if entries is None:
raise ValueError("Cannot find proper encoding for data file.")
df = pd.DataFrame(entries)
def converter(x):
try:
return ", ".join(x)
except TypeError:
return ""
for tag in LIST_TYPE_TAGS:
key = TAG_KEY_MAPPING[tag]
if key in df:
df[key] = df[key].apply(converter)
return standardize_dataframe(df)
def read_csv(data_fp):
"""CVS file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the CSV file.
Returns
-------
list:
List with entries.
"""
try:
df = pd.read_csv(data_fp)
except UnicodeDecodeError:
df = pd.read_csv(data_fp, encoding="ISO-8859-1")
return standardize_dataframe(df)
def read_ris(fp):
"""RIS file reader.
Parameters
----------
fp: str, pathlib.Path
File path to the RIS file.
label: bool
Check for label. If None, this is automatic.
Returns
-------
list:
List with entries.
"""
encodings = ['ISO-8859-1', 'utf-8', 'utf-8-sig']
entries = None
for encoding in encodings:
try:
with open(fp, 'r', encoding=encoding) as bibliography_file:
mapping = _tag_key_mapping(reverse=False)
entries = list(readris(bibliography_file, mapping=mapping))
break
except (UnicodeDecodeError, IOError):
pass
if entries is None:
raise ValueError("Cannot find proper encoding for data file.")
df = pd.DataFrame(entries)
if "keywords" in df:
def converter(x):
try:
return ", ".join(x)
except TypeError:
return ""
df["keywords"] = df["keywords"].apply(converter)
return standardize_dataframe(df)