def refine(search_path: str,
categories: Optional[dict] = None,
highlights: Optional[list] = None,
show_abstract: Optional[bool] = False,
show_extra_info: Optional[bool] = False,
only_selected_papers: Optional[bool] = False,
only_removed_papers: Optional[bool] = False,
read_only: Optional[bool] = False,
verbose: Optional[bool] = False):
"""
When you have a search result and wanna refine it, this is the method that you'll need to call.
This method will iterate through all the papers showing their collected data,
then asking if you wanna select a particular paper or not, and assign a category if a list of categories is provided.
And to help you on the refinement, this method can also highlight some terms on the paper's abstract by a provided list of them
Parameters
----------
search_path : str
valid file path containing a JSON representation of the search results
categories : dict, optional
A dict with lists of categories by their facets, used to assign to selected papers, by default None
E.g.:
{
'Research Type': [
'Validation Research', 'Evaluation Research', 'Solution Proposal', 'Philosophical', 'Opinion', 'Experience'
],
'Contribution': [
'Metric', 'Tool', 'Model', 'Method'
]
}
highlights : list, optional
A list of terms to highlight on the paper's abstract', by default None
show_abstract : bool, optional
A flag to indicate if the abstract should be shown or not, by default False
show_extra_info : bool, optional
A flag to indicate if the paper's extra info should be shown or not, by default False
only_selected_papers : bool, False by default
If only the selected papers will be refined, by default False
only_removed_papers : bool, False by default
If only the removed papers will be refined, by default False
read_only : bool, optional
If true, this method will only list the papers, by default False
verbose : Optional[bool], optional
If you wanna a verbose logging
"""
common_util.logging_initialize(verbose)
common_util.check_write_access(search_path)
init(autoreset=True) # colorama initializer
if categories is None:
categories = {}
if highlights is None:
highlights = []
search = persistence_util.load(search_path)
has_already_refined_papers = False
for paper in search.papers:
if paper.selected is not None:
has_already_refined_papers = True
break
todo_papers = []
done_papers = []
for paper in search.papers:
#if wanna_re_refine_papers:
if (only_selected_papers or only_removed_papers):
if paper.selected is not None and (
(only_selected_papers and paper.selected) or
(only_removed_papers and not paper.selected)):
todo_papers.append(paper)
else:
if paper.selected is None or read_only:
todo_papers.append(paper)
else:
done_papers.append(paper)
todo_papers = sorted(todo_papers,
key=lambda x: x.publication_date,
reverse=True)
for i, paper in enumerate(todo_papers):
print(f'\n{"." * os.get_terminal_size()[0]}\n')
if not read_only:
print(f'\n{Fore.CYAN}{i+1}/{len(todo_papers)} papers\n')
_print_paper_details(paper, highlights, show_abstract, show_extra_info)
if not read_only:
answer = _get_select_question_input()
if answer == 'Skip':
continue
elif answer == 'No':
paper.selected = False
elif answer == 'Yes':
paper.selected = True
else:
break
if paper.selected:
paper.categories = _get_category_question_input(categories)
done_papers.append(paper)
if read_only:
print(f'\n{Fore.CYAN}{len(todo_papers)} papers\n')
else:
persistence_util.save(search, search_path)
def bibtex(
filepath: str = typer.Argument(
..., help='A valid file path for the search result file'),
outputpath: str = typer.
Argument(
...,
help='A valid directory path where the generated bibtex will be placed'
),
only_selected_papers: bool = typer.
Option(
False,
"-s",
"--selected",
show_default=True,
help=
"A flag to indicate if only selected papers (selections be done on refine command) will be used for bibtex generation"
),
categories: List[str] = typer.
Option(
[],
"-c",
"--categories",
show_default=True,
help=
"A comma-separated list of categories (categorization can be done on refine command) that will be used to filter which papers will be used for bibtex generation, using the following pattern: <facet>:<term_b>,<term_c>,..."
),
add_findpapers_citation: bool = typer.
Option(
False,
"-f",
"--findpapers",
show_default=True,
help=
"A flag to indicate if you want to add an entry for Findpapers in your BibTeX output file"
),
verbose: bool = typer.Option(
False,
"-v",
"--verbose",
show_default=True,
help="If you wanna a verbose mode logging")):
"""
Generate a BibTeX file from the search results.
You can generate the bibtex only for the selected papers by using the -s (or --selected) flag
You can filter which kind of categorized papers will be used for bibtex generation providing
a comma-separated list of categories is provided by the -c (or --categories) argument,
You need to define these categories following the pattern: <facet>:<term_b>,<term_c>,...
E.g.:
--categories "Contribution:Metric,Tool"
The -c parameter can be defined several times, so you can define as many filters as you want.
The -c parameter is case-sensitive.
You can control the command logging verbosity by the -v (or --verbose) argument.
"""
try:
common_util.logging_initialize(verbose)
categories_by_facet = {} if len(categories) > 0 else None
for categories_string in categories:
string_split = categories_string.split(':')
facet = string_split[0].strip()
categories_by_facet[facet] = [
x.strip() for x in string_split[1].split(',')
]
findpapers.generate_bibtex(filepath, outputpath, only_selected_papers,
categories_by_facet,
add_findpapers_citation)
except Exception as e:
if verbose:
logging.debug(e, exc_info=True)
else:
typer.echo(e)
raise typer.Exit(code=1)
def search(outputpath: str, query: Optional[str] = None, since: Optional[datetime.date] = None, until: Optional[datetime.date] = None,
limit: Optional[int] = None, limit_per_database: Optional[int] = None, databases: Optional[List[str]] = None,
publication_types: Optional[List[str]] = None, scopus_api_token: Optional[str] = None, ieee_api_token: Optional[str] = None,
proxy: Optional[str] = None, verbose: Optional[bool] = False):
"""
When you have a query and needs to get papers using it, this is the method that you'll need to call.
This method will find papers from some databases based on the provided query.
Parameters
----------
outputpath : str
A valid file path where the search result file will be placed
query : str, optional
A query string that will be used to perform the papers search.
If not provided, the query will be loaded from the environment variable FINDPAPERS_QUERY
All the query terms need to be enclosed in quotes and can be associated using boolean operators,
and grouped using parentheses.
E.g.: [term A] AND ([term B] OR [term C]) AND NOT [term D]
You can use some wildcards in the query too. Use ? to replace a single character or * to replace any number of characters.
E.g.: "son?" -> will match song, sons, ...
E.g.: "son*" -> will match song, sons, sonar, songwriting, ...
Note: All boolean operators needs to be uppercased. The boolean operator "NOT" must be preceded by an "AND" operator.
since : Optional[datetime.date], optional
A lower bound (inclusive) date that will be used to filter the search results, by default None
until : Optional[datetime.date], optional
A upper bound (inclusive) date that will be used to filter the search results, by default None
limit : Optional[int], optional
The max number of papers to collect, by default None
limit_per_database : Optional[int], optional
The max number of papers to collect per each database, by default None
databases : List[str], optional
List of databases where the search should be performed, if not specified all databases will be used, by default None
publication_types : List[str], optional
List of publication list of publication types to filter when searching, if not specified all the publication types
will be collected (this parameter is case insensitive). The available publication types are: journal, conference proceedings, book, other, by default None
scopus_api_token : Optional[str], optional
A API token used to fetch data from Scopus database. If you don't have one go to https://dev.elsevier.com and get it, by default None
ieee_api_token : Optional[str], optional
A API token used to fetch data from IEEE database. If you don't have one go to https://developer.ieee.org and get it, by default None
proxy : Optional[str], optional
proxy URL that can be used during requests. This can be also defined by an environment variable FINDPAPERS_PROXY. By default None
verbose : Optional[bool], optional
If you wanna a verbose logging
"""
common_util.logging_initialize(verbose)
if proxy is not None:
os.environ['FINDPAPERS_PROXY'] = proxy
logging.info('Let\'s find some papers, this process may take a while...')
if databases is not None:
databases = [x.lower() for x in databases]
if publication_types is not None:
publication_types = [x.lower().strip() for x in publication_types]
for publication_type in publication_types:
if publication_type not in ['journal', 'conference proceedings', 'book', 'other']:
raise ValueError(f'Invalid publication type: {publication_type}')
if query is None:
query = os.getenv('FINDPAPERS_QUERY')
if query is not None:
query = _sanitize_query(query)
if query is None or not _is_query_ok(query):
raise ValueError('Invalid query format')
common_util.check_write_access(outputpath)
if ieee_api_token is None:
ieee_api_token = os.getenv('FINDPAPERS_IEEE_API_TOKEN')
if scopus_api_token is None:
scopus_api_token = os.getenv('FINDPAPERS_SCOPUS_API_TOKEN')
search = Search(query, since, until, limit, limit_per_database, databases=databases, publication_types=publication_types)
if databases is None or arxiv_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: arxiv_searcher.run(search),
search, arxiv_searcher.DATABASE_LABEL)
if databases is None or pubmed_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: pubmed_searcher.run(search),
search, pubmed_searcher.DATABASE_LABEL)
if databases is None or acm_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: acm_searcher.run(search),
search, acm_searcher.DATABASE_LABEL)
if ieee_api_token is not None:
if databases is None or ieee_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: ieee_searcher.run(
search, ieee_api_token), search, ieee_searcher.DATABASE_LABEL)
else:
logging.info('IEEE API token not found, skipping search on this database')
if scopus_api_token is not None:
if databases is None or scopus_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: scopus_searcher.run(
search, scopus_api_token), search, scopus_searcher.DATABASE_LABEL)
else:
logging.info('Scopus API token not found, skipping search on this database')
if databases is None or medrxiv_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: medrxiv_searcher.run(search),
search, medrxiv_searcher.DATABASE_LABEL)
if databases is None or biorxiv_searcher.DATABASE_LABEL.lower() in databases:
_database_safe_run(lambda: biorxiv_searcher.run(search),
search, biorxiv_searcher.DATABASE_LABEL)
logging.info('Enriching results...')
_enrich(search, scopus_api_token)
logging.info('Filtering results...')
_filter(search)
logging.info('Finding and merging duplications...')
search.merge_duplications()
logging.info('Flagging potentially predatory publications...')
_flag_potentially_predatory_publications(search)
logging.info(f'It\'s finally over! {len(search.papers)} papers retrieved. Good luck with your research :)')
persistence_util.save(search, outputpath)
def download(
filepath: str = typer.Argument(
..., help='A valid file path for the search result file'),
outputpath: str = typer.
Argument(
...,
help='A valid directory path where the downloaded papers will be placed'
),
only_selected_papers: bool = typer.
Option(
False,
"-s",
"--selected",
show_default=True,
help=
"A flag to indicate if only selected papers (selections can be done on refine command) will be downloaded"
),
categories: List[str] = typer.
Option(
[],
"-c",
"--categories",
show_default=True,
help=
"A comma-separated list of categories (categorization can be done on refine command) that will be used to filter which papers will be downloaded, using the following pattern: <facet>:<term_b>,<term_c>,..."
),
proxy: str = typer.Option(
None,
"-x",
"--proxy",
show_default=True,
help="proxy URL that can be used during requests"),
verbose: bool = typer.Option(
False,
"-v",
"--verbose",
show_default=True,
help="If you wanna a verbose mode logging")):
"""
Download full-text papers using the search results.
If you've done your search, (probably made the search refinement too) and wanna download the papers,
this is the command that you need to call. This command will try to download the PDF version of the papers to
the output directory path.
You can download only the selected papers by using the -s (or --selected) flag
You can filter which kind of categorized papers will be downloaded providing
a comma-separated list of categories is provided by the -c (or --categories) argument,
You need to define these categories following the pattern: <facet>:<term_b>,<term_c>,...
E.g.:
--categories "Contribution:Metric,Tool"
The -c parameter can be defined several times, so you can define as many filters as you want
The -c parameter is case-sensitive.
We use some heuristics to do our job, but sometime they won't work properly, and we cannot be able
to download the papers, but we logging the downloads or failures in a file download.log
placed on the output directory, you can check out the log to find what papers cannot be downloaded
and try to get them manually later.
Note: Some papers are behind a paywall and won't be able to be downloaded by this command.
However, if you have a proxy provided for the institution where you study or work that permit you
to "break" this paywall. You can use this proxy configuration here
by setting the environment variable FINDPAPERS_PROXY.
You can control the command logging verbosity by the -v (or --verbose) argument.
"""
try:
common_util.logging_initialize(verbose)
categories_by_facet = {} if len(categories) > 0 else None
for categories_string in categories:
string_split = categories_string.split(':')
facet = string_split[0].strip()
categories_by_facet[facet] = [
x.strip() for x in string_split[1].split(',')
]
findpapers.download(filepath, outputpath, only_selected_papers,
categories_by_facet, proxy)
except Exception as e:
if verbose:
logging.debug(e, exc_info=True)
else:
typer.echo(e)
raise typer.Exit(code=1)
def refine(
filepath: str = typer.Argument(
..., help='A valid file path for the search result file'),
categories: List[str] = typer.
Option(
[],
"-c",
"--categories",
show_default=True,
help=
"A comma-separated list of categories to assign to the papers with their facet following the pattern: <facet>:<term_b>,<term_c>,..."
),
highlights: str = typer.
Option(
None,
"-h",
"--highlights",
show_default=True,
help="A comma-separated list of terms to be highlighted on the abstract"
),
show_abstract: bool = typer.
Option(
False,
"-a",
"--abstract",
show_default=True,
help="A flag to indicate if the paper's abstract should be shown or not"
),
show_extra_info: bool = typer.
Option(
False,
"-e",
"--extra-info",
show_default=True,
help=
"A flag to indicate if the paper's extra info should be shown or not"),
only_selected_papers: bool = typer.Option(
False,
"-s",
"--selected",
show_default=True,
help="If only the selected papers will be refined"),
only_removed_papers: bool = typer.Option(
False,
"-r",
"--removed",
show_default=True,
help="If only the removed papers will be refined"),
read_only: bool = typer.
Option(
False,
"-l",
"--list",
show_default=True,
help=
"If this flag is present, this function call will only list the papers"
),
verbose: bool = typer.Option(
False,
"-v",
"--verbose",
show_default=True,
help="If you wanna a verbose mode logging")):
"""
Refine the search results by selecting/classifying the papers.
When you have a search result and wanna refine it, this is the command that you'll need to call.
This command will iterate through all the papers showing their collected data,
then asking if you wanna select a particular paper or not
You can show or hide the paper abstract by using the -a (or --abstract) flag.
If a comma-separated list of categories is provided by the -c (or --categories) argument,
you can assign a category to the paper. You need to define these categories following the pattern: <facet>:<term_b>,<term_c>,...
E.g.:
--categories "Contribution:Metric,Tool,Model,Method"
--categories "Research Type:Validation Research,Evaluation Research,Solution Proposal,Philosophical,Opinion,Experience"
The -c parameter can be defined several times, so you can define as many facets as you want
The -c parameter is case-sensitive.
And to help you on the refinement, this command can also highlight some terms on the paper's abstract
by a provided comma-separated list of them provided by the -h (or --highlights) argument.
You can control the command logging verbosity by the -v (or --verbose) argument.
"""
try:
common_util.logging_initialize(verbose)
highlights = [x.strip() for x in highlights.split(',')
] if highlights is not None else None
categories_by_facet = {} if len(categories) > 0 else None
for categories_string in categories:
string_split = categories_string.split(':')
facet = string_split[0].strip()
categories_by_facet[facet] = [
x.strip() for x in string_split[1].split(',')
]
findpapers.refine(filepath, categories_by_facet, highlights,
show_abstract, show_extra_info, only_selected_papers,
only_removed_papers, read_only)
except Exception as e:
if verbose:
logging.debug(e, exc_info=True)
else:
typer.echo(e)
raise typer.Exit(code=1)
def search(
outputpath: str = typer.
Argument(
...,
help=
'A valid file path where the search result JSON file will be placed'),
query: str = typer.
Option(
None,
"-q",
"--query",
show_default=True,
help=
'A query string that will be used to perform the papers search (If not provided it will be loaded from the environment variable FINDPAPERS_QUERY). E.g. [term A] AND ([term B] OR [term C]) AND NOT [term D]'
),
query_filepath: str = typer.
Option(
None,
"-f",
"--query-file",
show_default=True,
help=
'A file path that contains the query string that will be used to perform the papers search'
),
since: datetime = typer.
Option(
None,
"-s",
"--since",
show_default=True,
help=
"A lower bound (inclusive) date that will be used to filter the search results. Following the pattern YYYY-MM-DD. E.g. 2020-12-31",
formats=["%Y-%m-%d"]),
until: datetime = typer.
Option(
None,
"-u",
"--until",
show_default=True,
help=
"A upper bound (inclusive) date that will be used to filter the search results. Following the pattern YYYY-MM-DD. E.g. 2020-12-31",
formats=["%Y-%m-%d"]),
limit: int = typer.Option(None,
"-l",
"--limit",
show_default=True,
help="The max number of papers to collect"),
limit_per_database: int = typer.Option(
None,
"-ld",
"--limit-db",
show_default=True,
help="The max number of papers to collect per each database"),
databases: str = typer.
Option(
None,
"-d",
"--databases",
show_default=True,
help=
"A comma-separated list of databases where the search should be performed, if not specified all databases will be used (this parameter is case insensitive)"
),
publication_types: str = typer.
Option(
None,
"-p",
"--publication-types",
show_default=True,
help=
"A comma-separated list of publication types to filter when searching, if not specified all the publication types will be collected (this parameter is case insensitive). The available publication types are: journal, conference proceedings, book, other"
),
scopus_api_token: str = typer.
Option(
None,
"-ts",
"--token-scopus",
show_default=True,
help=
"A API token used to fetch data from Scopus database. If you don't have one go to https://dev.elsevier.com and get it. (If not provided it will be loaded from the environment variable FINDPAPERS_SCOPUS_API_TOKEN)"
),
ieee_api_token: str = typer.
Option(
None,
"-ti",
"--token-ieee",
show_default=True,
help=
"A API token used to fetch data from IEEE database. If you don't have one go to https://developer.ieee.org and get it. (If not provided it will be loaded from the environment variable FINDPAPERS_IEEE_API_TOKEN)"
),
proxy: str = typer.Option(
None,
"-x",
"--proxy",
show_default=True,
help="proxy URL that can be used during requests"),
verbose: bool = typer.Option(
False,
"-v",
"--verbose",
show_default=True,
help="If you wanna a verbose mode logging")):
"""
Search for papers metadata using a query.
When you have a query and needs to get papers using it, this is the command that you'll need to call.
This command will find papers from some databases based on the provided query.
All the query terms need to be enclosed by square brackets and can be associated using boolean operators,
and grouped using parentheses. The available boolean operators are "AND", "OR". "NOT".
All boolean operators needs to be uppercased. The boolean operator "NOT" must be preceded by an "AND" operator.
E.g.: [term A] AND ([term B] OR [term C]) AND NOT [term D]
You can use some wildcards in the query too. Use ? to replace a single character or * to replace any number of characters.
E.g.: 'son?' -> will match song, sons, ...
E.g.: 'son*' -> will match song, sons, sonar, songwriting, ...
Nowadays, we search for papers on ACM, arXiv, IEEE, PubMed, and Scopus database.
The searching on IEEE and Scopus requires an API token, that must to be provided
by the user using the -ts (or --scopus_api_token) and -te (or --ieee_api_token) arguments.
If these tokens are not provided the search on these databases will be skipped.
You can constraint the search by date using the -s (or --since) and -u (or --until) arguments
following the pattern YYYY-MM-DD (E.g. 2020-12-31).
You can restrict the max number of retrieved papers by using -l (or --limit).
And, restrict the max number of retrieved papers by database using -ld (or --limit_per_database) argument.
You can control which databases you would like to use in your search by the -d (or --databases) option. This parameter
accepts a comma-separated list of database names, and is case-insensitive. Nowadays the available databases are
ACM, arXiv, IEEE, PubMed, Scopus
E.g.:
--databases "scopus,arxiv,acm"
--databases "ieee,ACM,PubMed"
You can control which publication types you would like to fetch in your search by the -p (or --publication-types) option. This parameter
accepts a comma-separated list of database names, and is case-insensitive. Nowadays the available publication types are
journal, conference proceedings, book, other.
When a particular publication does not fit into any of the other types it is classified as "other", e.g., magazines, newsletters, unpublished manuscripts.
E.g.:
--publication-types "journal,conference proceedings,BOOK,other"
--publication-types "Journal,book"
You can control the command logging verbosity by the -v (or --verbose) argument.
"""
try:
since = since.date() if since is not None else None
until = until.date() if until is not None else None
databases = [x.strip() for x in databases.split(',')
] if databases is not None else None
publication_types = [x.strip() for x in publication_types.split(',')
] if publication_types is not None else None
common_util.logging_initialize(verbose)
if query is None and query_filepath is not None:
with open(query_filepath, 'r') as f:
query = f.read().strip()
findpapers.search(outputpath, query, since, until, limit,
limit_per_database, databases, publication_types,
scopus_api_token, ieee_api_token, proxy)
except Exception as e:
if verbose:
logging.debug(e, exc_info=True)
else:
typer.echo(e)
raise typer.Exit(code=1)
def download(search_path: str,
output_directory: str,
only_selected_papers: Optional[bool] = False,
categories_filter: Optional[dict] = None,
proxy: Optional[str] = None,
verbose: Optional[bool] = False):
"""
If you've done your search, (probably made the search refinement too) and wanna download the papers,
this is the method that you need to call. This method will try to download the PDF version of the papers to
the output directory path.
We use some heuristics to do our job, but sometime they won't work properly, and we cannot be able
to download the papers, but we logging the downloads or failures in a file download.log
placed on the output directory, you can check out the log to find what papers cannot be downloaded
and try to get them manually later.
Note: Some papers are behind a paywall and won't be able to be downloaded by this method.
However, if you have a proxy provided for the institution where you study or work that permit you
to "break" this paywall. You can use this proxy configuration here
by setting the environment variables FINDPAPERS_HTTP_PROXY and FINDPAPERS_HTTPS_PROXY.
Parameters
----------
search_path : str
A valid file path containing a JSON representation of the search results
output_directory : str
A valid file path of the directory where the downloaded papers will be placed
only_selected_papers : bool, False by default
If only the selected papers will be downloaded
categories_filter : dict, None by default
A dict of categories to be used to filter which papers will be downloaded
proxy : Optional[str], optional
proxy URL that can be used during requests. This can be also defined by an environment variable FINDPAPERS_PROXY. By default None
verbose : Optional[bool], optional
If you wanna a verbose logging
"""
common_util.logging_initialize(verbose)
if proxy is not None:
os.environ['FINDPAPERS_PROXY'] = proxy
search = persistence_util.load(search_path)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
log_filepath = os.path.join(output_directory, 'download.log')
common_util.check_write_access(log_filepath)
with open(log_filepath,
'a' if os.path.exists(log_filepath) else 'w') as fp:
now = datetime.datetime.now()
fp.write(
f"------- A new download process started at: {datetime.datetime.strftime(now, '%Y-%m-%d %H:%M:%S')} \n"
)
for i, paper in enumerate(search.papers):
logging.info(f'({i+1}/{len(search.papers)}) {paper.title}')
if (only_selected_papers and not paper.selected) or \
(categories_filter is not None and (paper.categories is None or not paper.has_category_match(categories_filter))):
continue
downloaded = False
output_filename = f'{paper.publication_date.year}-{paper.title}'
output_filename = re.sub(r'[^\w\d-]', '_',
output_filename) # sanitize filename
output_filename += '.pdf'
output_filepath = os.path.join(output_directory, output_filename)
if os.path.exists(output_filepath): # PDF already collected
logging.info(f'Paper\'s PDF file has already been collected')
continue
if paper.doi is not None:
paper.urls.add(f'http://doi.org/{paper.doi}')
for url in paper.urls: # we'll try to download the PDF file of the paper by its URLs
try:
logging.info(f'Fetching data from: {url}')
response = common_util.try_success(
lambda url=url: DefaultSession().get(url), 2)
if response is None:
continue
if 'text/html' in response.headers.get('content-type').lower():
response_url = urllib.parse.urlsplit(response.url)
response_query_string = urllib.parse.parse_qs(
urllib.parse.urlparse(response.url).query)
response_url_path = response_url.path
host_url = f'{response_url.scheme}://{response_url.hostname}'
pdf_url = None
if response_url_path.endswith('/'):
response_url_path = response_url_path[:-1]
response_url_path = response_url_path.split('?')[0]
if host_url in ['https://dl.acm.org']:
doi = paper.doi
if doi is None and response_url_path.startswith(
'/doi/'
) and '/doi/pdf/' not in response_url_path:
doi = response_url_path[4:]
elif doi is None:
continue
pdf_url = f'https://dl.acm.org/doi/pdf/{doi}'
elif host_url in ['https://ieeexplore.ieee.org']:
if response_url_path.startswith('/document/'):
document_id = response_url_path[10:]
elif response_query_string.get('arnumber',
None) is not None:
document_id = response_query_string.get(
'arnumber')[0]
else:
continue
pdf_url = f'{host_url}/stampPDF/getPDF.jsp?tp=&arnumber={document_id}'
elif host_url in [
'https://www.sciencedirect.com',
'https://linkinghub.elsevier.com'
]:
paper_id = response_url_path.split('/')[-1]
pdf_url = f'https://www.sciencedirect.com/science/article/pii/{paper_id}/pdfft?isDTMRedir=true&download=true'
elif host_url in ['https://pubs.rsc.org']:
pdf_url = response.url.replace('/articlelanding/',
'/articlepdf/')
elif host_url in [
'https://www.tandfonline.com',
'https://www.frontiersin.org'
]:
pdf_url = response.url.replace('/full', '/pdf')
elif host_url in [
'https://pubs.acs.org',
'https://journals.sagepub.com',
'https://royalsocietypublishing.org'
]:
pdf_url = response.url.replace('/doi', '/doi/pdf')
elif host_url in ['https://link.springer.com']:
pdf_url = response.url.replace(
'/article/', '/content/pdf/').replace('%2F',
'/') + '.pdf'
elif host_url in ['https://www.isca-speech.org']:
pdf_url = response.url.replace('/abstracts/',
'/pdfs/').replace(
'.html', '.pdf')
elif host_url in ['https://onlinelibrary.wiley.com']:
pdf_url = response.url.replace('/full/',
'/pdfdirect/').replace(
'/abs/',
'/pdfdirect/')
elif host_url in [
'https://www.jmir.org', 'https://www.mdpi.com'
]:
pdf_url = response.url + '/pdf'
elif host_url in ['https://www.pnas.org']:
pdf_url = response.url.replace(
'/content/', '/content/pnas/') + '.full.pdf'
elif host_url in ['https://www.jneurosci.org']:
pdf_url = response.url.replace(
'/content/', '/content/jneuro/') + '.full.pdf'
elif host_url in ['https://www.ijcai.org']:
paper_id = response.url.split('/')[-1].zfill(4)
pdf_url = '/'.join(response.url.split('/')
[:-1]) + '/' + paper_id + '.pdf'
elif host_url in [
'https://asmp-eurasipjournals.springeropen.com'
]:
pdf_url = response.url.replace('/articles/',
'/track/pdf/')
if pdf_url is not None:
response = common_util.try_success(
lambda url=pdf_url: DefaultSession().get(url), 2)
if 'application/pdf' in response.headers.get(
'content-type').lower():
with open(output_filepath, 'wb') as fp:
fp.write(response.content)
downloaded = True
break
except Exception as e: # pragma: no cover
logging.debug(e, exc_info=True)
if downloaded:
with open(log_filepath, 'a') as fp:
fp.write(f'[DOWNLOADED] {paper.title}\n')
else:
with open(log_filepath, 'a') as fp:
fp.write(f'[FAILED] {paper.title}\n')
if len(paper.urls) == 0:
fp.write(f'Empty URL list\n')
else:
for url in paper.urls:
fp.write(f'{url}\n')
def generate_bibtex(search_path: str,
outputpath: str,
only_selected_papers: Optional[bool] = False,
categories_filter: Optional[dict] = None,
add_findpapers_citation: Optional[bool] = False,
verbose: Optional[bool] = False):
"""
Method used to generate a BibTeX file from a search result
Parameters
----------
search_path : str
A valid file path containing a JSON representation of the search results
outputpath : str
A valid file path for the BibTeX output file
only_selected_papers : bool, optional
If you only want to generate a BibTeX file for selected papers, by default False
categories_filter : dict, None by default
A dict of categories to be used to filter which papers will be downloaded
add_findpapers_citation : bool, optional
If you want to add an entry for Findpapers in your BibTeX output file, by default False
verbose : Optional[bool], optional
If you wanna a verbose logging
"""
common_util.logging_initialize(verbose)
search = persistence_util.load(search_path)
common_util.check_write_access(outputpath)
default_tab = ' ' * 4
bibtex_output = ''
if add_findpapers_citation:
bibtex_output = '\n'.join([
'@misc{grosman2020findpapers', '\ttitle = {Findpapers},',
'\tauthor = {Grosman, Jonatas},', '\tpublisher = {GitHub},',
'\tjournal = {GitHub repository},',
'\thowpublished = {\\url{https://github.com/jonatasgrosman/findpapers}},',
'\tyear = {2020}', '}\n\n'
])
for paper in search.papers:
if (only_selected_papers and not paper.selected) or \
(categories_filter is not None and (paper.categories is None or not paper.has_category_match(categories_filter))):
continue
logging.info(f'Exporting bibtex for: {paper.title}')
try:
citation_type = '@unpublished'
if paper.publication is not None:
if paper.publication.category == 'Journal':
citation_type = '@article'
elif paper.publication.category == 'Conference Proceedings':
citation_type = '@inproceedings'
elif paper.publication.category == 'Book':
citation_type = '@book'
else:
citation_type = '@misc'
bibtex_output += f'{citation_type}{"{"}{paper.get_citation_key()},\n'
bibtex_output += f'{default_tab}title = {{{paper.title}}},\n'
if len(paper.authors) > 0:
authors = ' and '.join(paper.authors)
bibtex_output += f'{default_tab}author = {{{authors}}},\n'
if citation_type == '@unpublished':
note = ''
if len(paper.urls) > 0:
note += f'Available at {list(paper.urls)[0]}'
if paper.publication_date is not None:
note += f' ({paper.publication_date.strftime("%Y/%m/%d")})'
if paper.comments is not None:
note += paper.comments if len(
note) == 0 else f' | {paper.comments}'
bibtex_output += f'{default_tab}note = {{{note}}},\n'
elif citation_type == '@article':
bibtex_output += f'{default_tab}journal = {{{paper.publication.title}}},\n'
elif citation_type == '@inproceedings':
bibtex_output += f'{default_tab}booktitle = {{{paper.publication.title}}},\n'
elif citation_type == '@misc' and len(
paper.urls) > 0 and paper.publication_date is not None:
date = paper.publication_date.strftime('%Y/%m/%d')
url = list(paper.urls)[0]
bibtex_output += f'{default_tab}howpublished = {{Available at {url} ({date})}},\n'
if paper.publication is not None and paper.publication.publisher is not None:
bibtex_output += f'{default_tab}publisher = {{{paper.publication.publisher}}},\n'
if paper.publication_date is not None:
bibtex_output += f'{default_tab}year = {{{paper.publication_date.year}}},\n'
if paper.pages is not None:
bibtex_output += f'{default_tab}pages = {{{paper.pages}}},\n'
bibtex_output = bibtex_output.rstrip(
',\n') + '\n' # removing last comma
bibtex_output += '}\n\n'
except Exception as e:
logging.debug(e, exc_info=True)
with open(outputpath, 'w') as fp:
fp.write(bibtex_output)