def main(self): # TODO: Try to recycle some of this code with command add. documents = papis.api.get_documents_in_lib(self.get_args().lib, self.get_args().search) document = self.pick(documents) if not document: return 0 data = dict() if self.args.auto: if 'doi' in document.keys() and not self.args.from_doi: self.args.from_doi = document['doi'] if 'title' in document.keys() and not self.args.from_isbnplus: self.args.from_isbnplus = document['title'] if self.args.from_yaml: import yaml data.update(yaml.load(open(self.args.from_yaml))) if self.args.from_isbnplus: doc = self.pick([ papis.document.Document(data=d) for d in papis.isbn.get_data(query=self.args.from_isbnplus) ]) if doc: data.update(doc.to_dict()) if self.args.from_bibtex: bib_data = papis.bibtex.bibtex_to_dict(self.args.from_bibtex) if len(bib_data) > 1: self.logger.warning( 'Your bibtex file contains more than one entry,' ' I will be taking the first entry') data.update(bib_data[0]) if self.args.from_url: url_data = papis.downloaders.utils.get(self.args.from_url) data.update(url_data["data"]) document_paths = url_data["documents_paths"] if not len(document_paths) == 0: document_path = document_paths[0] old_doc = self.pick(document["files"]) if papis.utils.confirm("Really replace document %s?" % old_doc): new_path = os.path.join(document.get_main_folder(), old_doc) self.logger.debug("Moving %s to %s" % (document_path, new_path)) shutil.move(document_path, new_path) if self.args.from_doi: self.logger.debug("Try using doi %s" % self.args.from_doi) data.update(papis.utils.doi_to_data(self.args.from_doi)) document.update(data, self.args.force, self.args.interactive) document.save()
def locate_document(document, documents): """Try to figure out if a document is already within a list of documents. :param document: Document to be searched for :type document: papis.document.Document :param documents: Documents to search in :type documents: list :returns: TODO """ # if these keys exist in the documents, then check those first for d in documents: for key in ['doi', 'ref', 'isbn', 'isbn10', 'url']: if key in document.keys() and key in d.keys(): if document[key] == d[key]: return d # else, just try to match the usual way the documents # TODO: put this into the databases import papis.database.cache docs = papis.database.cache.filter_documents( documents, search='author = "{doc[author]}" title = "{doc[title]}"'.format( doc=document)) if len(docs) == 1: return docs[0] return None
def run(document: papis.document.Document, filepaths: List[str], git: bool = False) -> None: logger = logging.getLogger('addto') from string import ascii_lowercase g = papis.utils.create_identifier(ascii_lowercase) string_append = '' _doc_folder = document.get_main_folder() if not _doc_folder: raise Exception("Document does not have a folder attached") for i in range(len(document.get_files())): string_append = next(g) new_file_list = [] for i in range(len(filepaths)): in_file_path = filepaths[i] if not os.path.exists(in_file_path): raise Exception("{} not found".format(in_file_path)) # Rename the file in the staging area new_filename = papis.utils.clean_document_name( papis.commands.add.get_file_name(papis.document.to_dict(document), in_file_path, suffix=string_append)) new_file_list.append(new_filename) end_document_path = os.path.join(_doc_folder, new_filename) string_append = next(g) # Check if the absolute file path is > 255 characters if len(os.path.abspath(end_document_path)) >= 255: logger.warning('Length of absolute path is > 255 characters. ' 'This may cause some issues with some pdf viewers') if os.path.exists(end_document_path): logger.warning("%s already exists, ignoring...", end_document_path) continue import shutil logger.info("[CP] '%s' to '%s'", in_file_path, end_document_path) shutil.copy(in_file_path, end_document_path) if "files" not in document.keys(): document["files"] = [] document['files'] += new_file_list document.save() papis.database.get().update(document) if git: for r in new_file_list + [document.get_info_file()]: papis.git.add(_doc_folder, r) papis.git.commit( _doc_folder, "Add new files to '{}'".format(papis.document.describe(document)))
def locate_document(document, documents): """Try to figure out if a document is already within a list of documents. :param document: Document to be searched for :type document: papis.document.Document :param documents: Documents to search in :type documents: list :returns: papis document if it is found """ # if these keys exist in the documents, then check those first # TODO: find a way to really match well titles and author comparing_keys = eval(papis.config.get('unique-document-keys')) for d in documents: for key in comparing_keys: if key in document.keys() and key in d.keys(): if re.match(document[key], d[key], re.I): return d
def run(document, filepaths): logger = logging.getLogger('addto') g = papis.utils.create_identifier(ascii_lowercase) string_append = '' for i in range(len(document.get_files())): string_append = next(g) new_file_list = [] for i in range(len(filepaths)): in_file_path = filepaths[i] if not os.path.exists(in_file_path): raise Exception("{} not found".format(in_file_path)) # Rename the file in the staging area new_filename = papis.utils.clean_document_name( papis.commands.add.get_file_name(papis.document.to_dict(document), in_file_path, suffix=string_append)) new_file_list.append(new_filename) endDocumentPath = os.path.join(document.get_main_folder(), new_filename) string_append = next(g) # Check if the absolute file path is > 255 characters if len(os.path.abspath(endDocumentPath)) >= 255: logger.warning('Length of absolute path is > 255 characters. ' 'This may cause some issues with some pdf viewers') if os.path.exists(endDocumentPath): logger.warning("%s already exists, ignoring..." % endDocumentPath) continue logger.debug("[CP] '%s' to '%s'" % (in_file_path, endDocumentPath)) shutil.copy(in_file_path, endDocumentPath) if not "files" in document.keys(): document["files"] = [] document['files'] += new_file_list document.save() papis.database.get().update(document)
def locate_document(document, documents): """Try to figure out if a document is already within a list of documents. :param document: Document to be searched for :type document: papis.document.Document :param documents: Documents to search in :type documents: list :returns: TODO """ for d in documents: for key in ['doi', 'ref', 'isbn', 'isbn10', 'url']: if 'doi' in document.keys() and 'doi' in d.keys(): if document['doi'] == d['doi']: return d docs = filter_documents( documents, search='author = "{doc[author]}" title = "{doc[title]}"' ) if len(docs) == 1: return docs[0] return None
def cli(query, doc_folder, interactive, force, from_crossref, from_base, from_isbnplus, from_isbn, from_yaml, from_bibtex, from_url, from_doi, auto, all, set, delete): """Update a document from a given library""" documents = papis.database.get().query(query) logger = logging.getLogger('cli:update') if not documents: logger.warning(papis.strings.no_documents_retrieved_message) if doc_folder: documents = [papis.document.from_folder(doc_folder)] if not all: documents = filter(lambda d: d, [papis.api.pick_doc(documents)]) for document in documents: data = dict() logger.info( 'Updating ' '{c.Back.WHITE}{c.Fore.BLACK}{0}{c.Style.RESET_ALL}'.format( papis.document.describe(document), c=colorama)) if set: data.update( {s[0]: papis.utils.format_doc(s[1], document) for s in set}) if delete: for key in delete: _delete_key = False _confirmation = True if interactive: _confirmation = papis.utils.confirm( "Delete {key}?".format(key=key)) if interactive and _confirmation and not force: _delete_key = True elif not _confirmation: _delete_key = False else: _delete_key = True if _delete_key: try: logger.warning('Deleting {key}'.format(key=key)) del document[key] except ValueError: logger.error('Document has no {key}'.format(key=key)) else: _update_with_database(document) if auto: if 'doi' in document.keys() and not from_doi: logger.info('Trying using the doi {}'.format(document['doi'])) from_doi = document['doi'] if 'url' in document.keys() and not from_url: logger.info('Trying using the url {}'.format(document['url'])) from_url = document['url'] if 'title' in document.keys() and not from_isbn: from_isbn = '{d[title]} {d[author]}'.format(d=document) from_isbnplus = from_isbn from_base = from_isbn logger.info( 'Trying with `from_isbn`, `from_isbnplus` and `from_base` ' 'with the text "{0}"'.format(from_isbn)) if from_crossref is None and from_doi is None: from_crossref = True if from_crossref: query = papis.utils.format_doc(from_crossref, document) logger.info('Trying with crossref with query {0}'.query(query)) if from_crossref is True: from_crossref = '' try: doc = papis.api.pick_doc([ papis.document.from_data(d) for d in papis.crossref.get_data(query=query, author=document['author'], title=document['title']) ]) if doc: data.update(papis.document.to_dict(doc)) if 'doi' in document.keys() and not from_doi and auto: from_doi = doc['doi'] except Exception as e: logger.error('error processing crossref') logger.error(e) if from_base: query = papis.utils.format_doc(from_base, document) logger.info('Trying with base with query {0}'.format(query)) try: doc = papis.api.pick_doc([ papis.document.from_data(d) for d in papis.base.get_data(query=query) ]) if doc: data.update(papis.document.to_dict(doc)) except urllib.error.HTTPError: logger.error('urllib failed to download') if from_isbnplus: logger.info('Trying with isbnplus') logger.warning('Isbnplus support is does not work... Not my fault') if from_isbn: query = papis.utils.format_doc(from_isbn, document) logger.info('Trying with isbn ({0:20})'.format(query)) try: doc = papis.api.pick_doc([ papis.document.from_data(d) for d in papis.isbn.get_data(query=query) ]) if doc: data.update(papis.document.to_dict(doc)) except Exception as e: logger.error('Isbnlib had an error retrieving information') logger.error(e) if from_yaml: data.update(papis.yaml.yaml_to_data(from_yaml)) if from_doi: query = papis.utils.format_doc(from_doi, document) logger.info("Try using doi %s" % query) doidata = papis.crossref.doi_to_data(query) if doidata: data.update(doidata) if from_bibtex: try: bib_data = papis.bibtex.bibtex_to_dict(from_bibtex) data.update(bib_data[0]) except Exception as e: logger.error('error processing bibtex') logger.error(e) if from_url: query = papis.utils.format_doc(from_url, document) logger.info('Trying url {0}'.format(query)) try: url_data = papis.downloaders.get_info_from_url(query) data.update(url_data["data"]) except Exception as e: logger.error('error processing url') logger.error(e) run( document, data=data, interactive=interactive, force=force, )
def run(document=papis.document.Document(), file_list=[], importer_list=(), file_format=None, folder_format=None, ref_format=None, confirm_flag=True, edit_flag=True, suffix_flag=True): log_prefix = "add:{0}".format("document") logger = logging.getLogger(log_prefix) if file_format is None and user._add_file_name: file_format = user._add_file_name if folder_format is None and user._add_folder_name: folder_format = user._add_folder_name if ref_format is None and user._add_ref_name: ref_format = user._add_ref_name file_list = papis.utils._get_list(file_list) ctx = get_data_from_importer(importer_list) for key in ctx.data.keys(): document.set_key(key, ctx.data[key], append=True) if papis.utils._validate_list(ctx.files): file_list.append(ctx.files[0]) if not document.keys(): logger.error(log.no_data_retrieved) # @log return 1 if confirm_flag: data = papis.document.from_input( document, "The following document will be added to your library") if not data: return 1 else: document.update(data) folder_name, document = papis.support.identifier.get_identifier( data=document, file_list=file_list, folder_format=folder_format, file_format=file_format, ref_format=ref_format, suffix=suffix_flag) lib_mgr = papis.libmanager.get_lib_manager(logger=logger) document = lib_mgr.add(document=document, folder_name=folder_name, value=file_list, doc=True, file_name=document[user._files_name]) if edit_flag: from papis.commands.open import run doc_mgr = lib_mgr.set_doc_item(doc_list=[document], fun_item=task.get_item( task._task_editor), fun_mark=None, pick=False) run(doc_mgr, open_task=task._task_editor, verbose=False) return document
def main(self): # TODO: Try to recycle some of this code with command add. import papis.api documents = papis.api.get_documents_in_lib(self.get_args().lib, self.get_args().search) data = dict() if self.args.from_bibtex: bib_data = papis.bibtex.bibtex_to_dict(self.args.from_bibtex) # Then it means that the user wants to update all the information # appearing in the bibtex file if len(bib_data) > 1: self.logger.warning( 'Your bibtex file contains more than one entry,') self.logger.warning( 'It is supposed that you want to update all the' 'documents appearing inside the bibtex file.') for bib_element in bib_data: doc = papis.document.Document(data=bib_element) located_doc = papis.utils.locate_document(doc, documents) if located_doc is None: self.logger.error( "The following information could not be located") self.logger.error('\n' + doc.dump()) else: located_doc.update(bib_element, self.args.force, self.args.interactive) located_doc.save() return 0 data.update(bib_data[0]) if self.args.from_yaml: import yaml data.update(yaml.load(open(self.args.from_yaml))) # For the coming parts we need to pick a document document = self.pick(documents) if not document: return 0 if self.args.auto: if 'doi' in document.keys() and not self.args.from_doi: self.args.from_doi = document['doi'] if 'title' in document.keys() and not self.args.from_isbnplus: self.args.from_isbnplus = document['title'] if self.args.from_isbnplus: import papis.isbn doc = self.pick([ papis.document.Document(data=d) for d in papis.isbn.get_data(query=self.args.from_isbnplus) ]) if doc: data.update(doc.to_dict()) if self.args.from_url: url_data = papis.downloaders.utils.get(self.args.from_url) data.update(url_data["data"]) document_paths = url_data["documents_paths"] if not len(document_paths) == 0: document_path = document_paths[0] old_doc = self.pick(document["files"]) if papis.utils.confirm("Really replace document %s?" % old_doc): new_path = os.path.join(document.get_main_folder(), old_doc) self.logger.debug("Moving %s to %s" % (document_path, new_path)) shutil.move(document_path, new_path) if self.args.from_doi: self.logger.debug("Try using doi %s" % self.args.from_doi) data.update(papis.utils.doi_to_data(self.args.from_doi)) document.update(data, self.args.force, self.args.interactive) document.save()
def cli( query, interactive, force, from_crossref, from_base, from_isbnplus, from_isbn, from_yaml, from_bibtex, from_url, from_doi, auto, all, set ): """Update a document from a given library""" documents = papis.database.get().query(query) data = dict() logger = logging.getLogger('cli:update') if not all: documents = [papis.api.pick_doc(documents)] for document in documents: if all: data = dict() from_url = None from_doi = None from_isbnplus = None from_isbnplus = None if set: data.update( {s[0]: papis.utils.format_doc(s[1], document) for s in set} ) if auto: if 'doi' in document.keys() and not from_doi: logger.info('Trying using the doi {}'.format(document['doi'])) from_doi = document['doi'] if 'url' in document.keys() and not from_url: logger.info('Trying using the url {}'.format(document['url'])) from_url = document['url'] if 'title' in document.keys() and not from_isbn: from_isbn = '{d[title]} {d[author]}'.format(d=document) from_isbnplus = from_isbn from_base = from_isbn logger.info( 'Trying with `from_isbn`, `from_isbnplus` and `from_base` ' 'with the text "{0}"'.format(from_isbn) ) if from_crossref is None and from_doi is None: from_crossref = True if from_crossref: logger.info('Trying with crossref') if from_crossref is True: from_crossref = '' try: doc = papis.api.pick_doc([ papis.document.from_data(d) for d in papis.crossref.get_data( query=from_crossref, author=document['author'], title=document['title'] ) ]) if doc: data.update(papis.document.to_dict(doc)) if 'doi' in document.keys() and not from_doi and auto: from_doi = doc['doi'] except Exception as e: logger.error(e) if from_base: logger.info('Trying with base') try: doc = papis.api.pick_doc( [ papis.document.from_data(d) for d in papis.base.get_data( query=from_isbnplus ) ] ) if doc: data.update(papis.document.to_dict(doc)) except urllib.error.HTTPError: logger.error('urllib failed to download') if from_isbnplus: logger.info('Trying with isbnplus') logger.warning('Isbnplus does not work... Not my fault') # try: # doc = papis.api.pick_doc( # [ # papis.document.from_data(d) # for d in papis.isbnplus.get_data( # query=from_isbnplus # ) # ] # ) # if doc: # data.update(papis.document.to_dict(doc)) # except urllib.error.HTTPError: # logger.error('urllib failed to download') if from_isbn: logger.info('Trying with isbn') try: doc = papis.api.pick_doc( [ papis.document.from_data(d) for d in papis.isbn.get_data( query=from_isbn ) ] ) if doc: data.update(papis.document.to_dict(doc)) except Exception as e: logger.error('Isbnlib had an error retrieving information') logger.error(e) if from_yaml: with open(from_yaml) as fd: data.update(yaml.load(fd)) if from_doi: logger.debug("Try using doi %s" % from_doi) data.update(papis.utils.doi_to_data(from_doi)) if from_bibtex: try: bib_data = papis.bibtex.bibtex_to_dict(from_bibtex) data.update(bib_data[0]) except Exception: pass if from_url: logger.info('Trying url {0}'.format(from_url)) try: url_data = papis.downloaders.utils.get(from_url) data.update(url_data["data"]) except Exception: pass run( document, data=data, interactive=interactive, force=force, )