def main(self):
# TODO: Try to recycle some of this code with command add.
documents = papis.api.get_documents_in_lib(self.get_args().lib,
self.get_args().search)
document = self.pick(documents)
if not document: return 0
data = dict()
if self.args.auto:
if 'doi' in document.keys() and not self.args.from_doi:
self.args.from_doi = document['doi']
if 'title' in document.keys() and not self.args.from_isbnplus:
self.args.from_isbnplus = document['title']
if self.args.from_yaml:
import yaml
data.update(yaml.load(open(self.args.from_yaml)))
if self.args.from_isbnplus:
doc = self.pick([
papis.document.Document(data=d)
for d in papis.isbn.get_data(query=self.args.from_isbnplus)
])
if doc:
data.update(doc.to_dict())
if self.args.from_bibtex:
bib_data = papis.bibtex.bibtex_to_dict(self.args.from_bibtex)
if len(bib_data) > 1:
self.logger.warning(
'Your bibtex file contains more than one entry,'
' I will be taking the first entry')
data.update(bib_data[0])
if self.args.from_url:
url_data = papis.downloaders.utils.get(self.args.from_url)
data.update(url_data["data"])
document_paths = url_data["documents_paths"]
if not len(document_paths) == 0:
document_path = document_paths[0]
old_doc = self.pick(document["files"])
if papis.utils.confirm("Really replace document %s?" %
old_doc):
new_path = os.path.join(document.get_main_folder(),
old_doc)
self.logger.debug("Moving %s to %s" %
(document_path, new_path))
shutil.move(document_path, new_path)
if self.args.from_doi:
self.logger.debug("Try using doi %s" % self.args.from_doi)
data.update(papis.utils.doi_to_data(self.args.from_doi))
document.update(data, self.args.force, self.args.interactive)
document.save()
def locate_document(document, documents):
"""Try to figure out if a document is already within a list of documents.
:param document: Document to be searched for
:type document: papis.document.Document
:param documents: Documents to search in
:type documents: list
:returns: TODO
"""
# if these keys exist in the documents, then check those first
for d in documents:
for key in ['doi', 'ref', 'isbn', 'isbn10', 'url']:
if key in document.keys() and key in d.keys():
if document[key] == d[key]:
return d
# else, just try to match the usual way the documents
# TODO: put this into the databases
import papis.database.cache
docs = papis.database.cache.filter_documents(
documents,
search='author = "{doc[author]}" title = "{doc[title]}"'.format(
doc=document))
if len(docs) == 1:
return docs[0]
return None
def run(document: papis.document.Document,
filepaths: List[str],
git: bool = False) -> None:
logger = logging.getLogger('addto')
from string import ascii_lowercase
g = papis.utils.create_identifier(ascii_lowercase)
string_append = ''
_doc_folder = document.get_main_folder()
if not _doc_folder:
raise Exception("Document does not have a folder attached")
for i in range(len(document.get_files())):
string_append = next(g)
new_file_list = []
for i in range(len(filepaths)):
in_file_path = filepaths[i]
if not os.path.exists(in_file_path):
raise Exception("{} not found".format(in_file_path))
# Rename the file in the staging area
new_filename = papis.utils.clean_document_name(
papis.commands.add.get_file_name(papis.document.to_dict(document),
in_file_path,
suffix=string_append))
new_file_list.append(new_filename)
end_document_path = os.path.join(_doc_folder, new_filename)
string_append = next(g)
# Check if the absolute file path is > 255 characters
if len(os.path.abspath(end_document_path)) >= 255:
logger.warning('Length of absolute path is > 255 characters. '
'This may cause some issues with some pdf viewers')
if os.path.exists(end_document_path):
logger.warning("%s already exists, ignoring...", end_document_path)
continue
import shutil
logger.info("[CP] '%s' to '%s'", in_file_path, end_document_path)
shutil.copy(in_file_path, end_document_path)
if "files" not in document.keys():
document["files"] = []
document['files'] += new_file_list
document.save()
papis.database.get().update(document)
if git:
for r in new_file_list + [document.get_info_file()]:
papis.git.add(_doc_folder, r)
papis.git.commit(
_doc_folder,
"Add new files to '{}'".format(papis.document.describe(document)))
def locate_document(document, documents):
"""Try to figure out if a document is already within a list of documents.
:param document: Document to be searched for
:type document: papis.document.Document
:param documents: Documents to search in
:type documents: list
:returns: papis document if it is found
"""
# if these keys exist in the documents, then check those first
# TODO: find a way to really match well titles and author
comparing_keys = eval(papis.config.get('unique-document-keys'))
for d in documents:
for key in comparing_keys:
if key in document.keys() and key in d.keys():
if re.match(document[key], d[key], re.I):
return d
def run(document, filepaths):
logger = logging.getLogger('addto')
g = papis.utils.create_identifier(ascii_lowercase)
string_append = ''
for i in range(len(document.get_files())):
string_append = next(g)
new_file_list = []
for i in range(len(filepaths)):
in_file_path = filepaths[i]
if not os.path.exists(in_file_path):
raise Exception("{} not found".format(in_file_path))
# Rename the file in the staging area
new_filename = papis.utils.clean_document_name(
papis.commands.add.get_file_name(papis.document.to_dict(document),
in_file_path,
suffix=string_append))
new_file_list.append(new_filename)
endDocumentPath = os.path.join(document.get_main_folder(),
new_filename)
string_append = next(g)
# Check if the absolute file path is > 255 characters
if len(os.path.abspath(endDocumentPath)) >= 255:
logger.warning('Length of absolute path is > 255 characters. '
'This may cause some issues with some pdf viewers')
if os.path.exists(endDocumentPath):
logger.warning("%s already exists, ignoring..." % endDocumentPath)
continue
logger.debug("[CP] '%s' to '%s'" % (in_file_path, endDocumentPath))
shutil.copy(in_file_path, endDocumentPath)
if not "files" in document.keys():
document["files"] = []
document['files'] += new_file_list
document.save()
papis.database.get().update(document)
def locate_document(document, documents):
"""Try to figure out if a document is already within a list of documents.
:param document: Document to be searched for
:type document: papis.document.Document
:param documents: Documents to search in
:type documents: list
:returns: TODO
"""
for d in documents:
for key in ['doi', 'ref', 'isbn', 'isbn10', 'url']:
if 'doi' in document.keys() and 'doi' in d.keys():
if document['doi'] == d['doi']:
return d
docs = filter_documents(
documents, search='author = "{doc[author]}" title = "{doc[title]}"'
)
if len(docs) == 1:
return docs[0]
return None
def cli(query, doc_folder, interactive, force, from_crossref, from_base,
from_isbnplus, from_isbn, from_yaml, from_bibtex, from_url, from_doi,
auto, all, set, delete):
"""Update a document from a given library"""
documents = papis.database.get().query(query)
logger = logging.getLogger('cli:update')
if not documents:
logger.warning(papis.strings.no_documents_retrieved_message)
if doc_folder:
documents = [papis.document.from_folder(doc_folder)]
if not all:
documents = filter(lambda d: d, [papis.api.pick_doc(documents)])
for document in documents:
data = dict()
logger.info(
'Updating '
'{c.Back.WHITE}{c.Fore.BLACK}{0}{c.Style.RESET_ALL}'.format(
papis.document.describe(document), c=colorama))
if set:
data.update(
{s[0]: papis.utils.format_doc(s[1], document)
for s in set})
if delete:
for key in delete:
_delete_key = False
_confirmation = True
if interactive:
_confirmation = papis.utils.confirm(
"Delete {key}?".format(key=key))
if interactive and _confirmation and not force:
_delete_key = True
elif not _confirmation:
_delete_key = False
else:
_delete_key = True
if _delete_key:
try:
logger.warning('Deleting {key}'.format(key=key))
del document[key]
except ValueError:
logger.error('Document has no {key}'.format(key=key))
else:
_update_with_database(document)
if auto:
if 'doi' in document.keys() and not from_doi:
logger.info('Trying using the doi {}'.format(document['doi']))
from_doi = document['doi']
if 'url' in document.keys() and not from_url:
logger.info('Trying using the url {}'.format(document['url']))
from_url = document['url']
if 'title' in document.keys() and not from_isbn:
from_isbn = '{d[title]} {d[author]}'.format(d=document)
from_isbnplus = from_isbn
from_base = from_isbn
logger.info(
'Trying with `from_isbn`, `from_isbnplus` and `from_base` '
'with the text "{0}"'.format(from_isbn))
if from_crossref is None and from_doi is None:
from_crossref = True
if from_crossref:
query = papis.utils.format_doc(from_crossref, document)
logger.info('Trying with crossref with query {0}'.query(query))
if from_crossref is True:
from_crossref = ''
try:
doc = papis.api.pick_doc([
papis.document.from_data(d)
for d in papis.crossref.get_data(query=query,
author=document['author'],
title=document['title'])
])
if doc:
data.update(papis.document.to_dict(doc))
if 'doi' in document.keys() and not from_doi and auto:
from_doi = doc['doi']
except Exception as e:
logger.error('error processing crossref')
logger.error(e)
if from_base:
query = papis.utils.format_doc(from_base, document)
logger.info('Trying with base with query {0}'.format(query))
try:
doc = papis.api.pick_doc([
papis.document.from_data(d)
for d in papis.base.get_data(query=query)
])
if doc:
data.update(papis.document.to_dict(doc))
except urllib.error.HTTPError:
logger.error('urllib failed to download')
if from_isbnplus:
logger.info('Trying with isbnplus')
logger.warning('Isbnplus support is does not work... Not my fault')
if from_isbn:
query = papis.utils.format_doc(from_isbn, document)
logger.info('Trying with isbn ({0:20})'.format(query))
try:
doc = papis.api.pick_doc([
papis.document.from_data(d)
for d in papis.isbn.get_data(query=query)
])
if doc:
data.update(papis.document.to_dict(doc))
except Exception as e:
logger.error('Isbnlib had an error retrieving information')
logger.error(e)
if from_yaml:
data.update(papis.yaml.yaml_to_data(from_yaml))
if from_doi:
query = papis.utils.format_doc(from_doi, document)
logger.info("Try using doi %s" % query)
doidata = papis.crossref.doi_to_data(query)
if doidata:
data.update(doidata)
if from_bibtex:
try:
bib_data = papis.bibtex.bibtex_to_dict(from_bibtex)
data.update(bib_data[0])
except Exception as e:
logger.error('error processing bibtex')
logger.error(e)
if from_url:
query = papis.utils.format_doc(from_url, document)
logger.info('Trying url {0}'.format(query))
try:
url_data = papis.downloaders.get_info_from_url(query)
data.update(url_data["data"])
except Exception as e:
logger.error('error processing url')
logger.error(e)
run(
document,
data=data,
interactive=interactive,
force=force,
)
def run(document=papis.document.Document(),
file_list=[],
importer_list=(),
file_format=None,
folder_format=None,
ref_format=None,
confirm_flag=True,
edit_flag=True,
suffix_flag=True):
log_prefix = "add:{0}".format("document")
logger = logging.getLogger(log_prefix)
if file_format is None and user._add_file_name:
file_format = user._add_file_name
if folder_format is None and user._add_folder_name:
folder_format = user._add_folder_name
if ref_format is None and user._add_ref_name:
ref_format = user._add_ref_name
file_list = papis.utils._get_list(file_list)
ctx = get_data_from_importer(importer_list)
for key in ctx.data.keys():
document.set_key(key, ctx.data[key], append=True)
if papis.utils._validate_list(ctx.files):
file_list.append(ctx.files[0])
if not document.keys():
logger.error(log.no_data_retrieved) # @log
return 1
if confirm_flag:
data = papis.document.from_input(
document, "The following document will be added to your library")
if not data:
return 1
else:
document.update(data)
folder_name, document = papis.support.identifier.get_identifier(
data=document,
file_list=file_list,
folder_format=folder_format,
file_format=file_format,
ref_format=ref_format,
suffix=suffix_flag)
lib_mgr = papis.libmanager.get_lib_manager(logger=logger)
document = lib_mgr.add(document=document,
folder_name=folder_name,
value=file_list,
doc=True,
file_name=document[user._files_name])
if edit_flag:
from papis.commands.open import run
doc_mgr = lib_mgr.set_doc_item(doc_list=[document],
fun_item=task.get_item(
task._task_editor),
fun_mark=None,
pick=False)
run(doc_mgr, open_task=task._task_editor, verbose=False)
return document
def main(self):
# TODO: Try to recycle some of this code with command add.
import papis.api
documents = papis.api.get_documents_in_lib(self.get_args().lib,
self.get_args().search)
data = dict()
if self.args.from_bibtex:
bib_data = papis.bibtex.bibtex_to_dict(self.args.from_bibtex)
# Then it means that the user wants to update all the information
# appearing in the bibtex file
if len(bib_data) > 1:
self.logger.warning(
'Your bibtex file contains more than one entry,')
self.logger.warning(
'It is supposed that you want to update all the'
'documents appearing inside the bibtex file.')
for bib_element in bib_data:
doc = papis.document.Document(data=bib_element)
located_doc = papis.utils.locate_document(doc, documents)
if located_doc is None:
self.logger.error(
"The following information could not be located")
self.logger.error('\n' + doc.dump())
else:
located_doc.update(bib_element, self.args.force,
self.args.interactive)
located_doc.save()
return 0
data.update(bib_data[0])
if self.args.from_yaml:
import yaml
data.update(yaml.load(open(self.args.from_yaml)))
# For the coming parts we need to pick a document
document = self.pick(documents)
if not document: return 0
if self.args.auto:
if 'doi' in document.keys() and not self.args.from_doi:
self.args.from_doi = document['doi']
if 'title' in document.keys() and not self.args.from_isbnplus:
self.args.from_isbnplus = document['title']
if self.args.from_isbnplus:
import papis.isbn
doc = self.pick([
papis.document.Document(data=d)
for d in papis.isbn.get_data(query=self.args.from_isbnplus)
])
if doc:
data.update(doc.to_dict())
if self.args.from_url:
url_data = papis.downloaders.utils.get(self.args.from_url)
data.update(url_data["data"])
document_paths = url_data["documents_paths"]
if not len(document_paths) == 0:
document_path = document_paths[0]
old_doc = self.pick(document["files"])
if papis.utils.confirm("Really replace document %s?" %
old_doc):
new_path = os.path.join(document.get_main_folder(),
old_doc)
self.logger.debug("Moving %s to %s" %
(document_path, new_path))
shutil.move(document_path, new_path)
if self.args.from_doi:
self.logger.debug("Try using doi %s" % self.args.from_doi)
data.update(papis.utils.doi_to_data(self.args.from_doi))
document.update(data, self.args.force, self.args.interactive)
document.save()
def cli(
query,
interactive,
force,
from_crossref,
from_base,
from_isbnplus,
from_isbn,
from_yaml,
from_bibtex,
from_url,
from_doi,
auto,
all,
set
):
"""Update a document from a given library"""
documents = papis.database.get().query(query)
data = dict()
logger = logging.getLogger('cli:update')
if not all:
documents = [papis.api.pick_doc(documents)]
for document in documents:
if all:
data = dict()
from_url = None
from_doi = None
from_isbnplus = None
from_isbnplus = None
if set:
data.update(
{s[0]: papis.utils.format_doc(s[1], document) for s in set}
)
if auto:
if 'doi' in document.keys() and not from_doi:
logger.info('Trying using the doi {}'.format(document['doi']))
from_doi = document['doi']
if 'url' in document.keys() and not from_url:
logger.info('Trying using the url {}'.format(document['url']))
from_url = document['url']
if 'title' in document.keys() and not from_isbn:
from_isbn = '{d[title]} {d[author]}'.format(d=document)
from_isbnplus = from_isbn
from_base = from_isbn
logger.info(
'Trying with `from_isbn`, `from_isbnplus` and `from_base` '
'with the text "{0}"'.format(from_isbn)
)
if from_crossref is None and from_doi is None:
from_crossref = True
if from_crossref:
logger.info('Trying with crossref')
if from_crossref is True:
from_crossref = ''
try:
doc = papis.api.pick_doc([
papis.document.from_data(d)
for d in papis.crossref.get_data(
query=from_crossref,
author=document['author'],
title=document['title']
)
])
if doc:
data.update(papis.document.to_dict(doc))
if 'doi' in document.keys() and not from_doi and auto:
from_doi = doc['doi']
except Exception as e:
logger.error(e)
if from_base:
logger.info('Trying with base')
try:
doc = papis.api.pick_doc(
[
papis.document.from_data(d)
for d in papis.base.get_data(
query=from_isbnplus
)
]
)
if doc:
data.update(papis.document.to_dict(doc))
except urllib.error.HTTPError:
logger.error('urllib failed to download')
if from_isbnplus:
logger.info('Trying with isbnplus')
logger.warning('Isbnplus does not work... Not my fault')
# try:
# doc = papis.api.pick_doc(
# [
# papis.document.from_data(d)
# for d in papis.isbnplus.get_data(
# query=from_isbnplus
# )
# ]
# )
# if doc:
# data.update(papis.document.to_dict(doc))
# except urllib.error.HTTPError:
# logger.error('urllib failed to download')
if from_isbn:
logger.info('Trying with isbn')
try:
doc = papis.api.pick_doc(
[
papis.document.from_data(d)
for d in papis.isbn.get_data(
query=from_isbn
)
]
)
if doc:
data.update(papis.document.to_dict(doc))
except Exception as e:
logger.error('Isbnlib had an error retrieving information')
logger.error(e)
if from_yaml:
with open(from_yaml) as fd:
data.update(yaml.load(fd))
if from_doi:
logger.debug("Try using doi %s" % from_doi)
data.update(papis.utils.doi_to_data(from_doi))
if from_bibtex:
try:
bib_data = papis.bibtex.bibtex_to_dict(from_bibtex)
data.update(bib_data[0])
except Exception:
pass
if from_url:
logger.info('Trying url {0}'.format(from_url))
try:
url_data = papis.downloaders.utils.get(from_url)
data.update(url_data["data"])
except Exception:
pass
run(
document,
data=data,
interactive=interactive,
force=force,
)
