Exemplo n.º 1
0
class ZoteroBackend(object):
    @staticmethod
    def create_api_key():
        """ Interactively create a new API key via Zotero's OAuth API.

        Requires the user to enter a verification key displayed in the browser.

        :returns:   API key and the user's library ID
        """
        auth = OAuth1Service(name='zotero',
                             consumer_key=CLIENT_KEY,
                             consumer_secret=CLIENT_SECRET,
                             request_token_url=REQUEST_TOKEN_URL,
                             access_token_url=ACCESS_TOKEN_URL,
                             authorize_url=AUTH_URL,
                             base_url=BASE_URL)
        token, secret = auth.get_request_token(
            params={'oauth_callback': 'oob'})
        auth_url = auth.get_authorize_url(token)
        auth_url += '&' + urlencode({
            'name': 'zotero-cli',
            'library_access': 1,
            'notes_access': 1,
            'write_access': 1,
            'all_groups': 'read'
        })
        click.echo("Opening {} in browser, please confirm.".format(auth_url))
        click.launch(auth_url)
        verification = click.prompt("Enter verification code")
        token_resp = auth.get_raw_access_token(
            token,
            secret,
            method='POST',
            data={'oauth_verifier': verification})
        if not token_resp:
            logging.debug(token_resp.content)
            click.fail("Error during API key generation.")
        access = urlparse.parse_qs(token_resp.text)
        return access['oauth_token'][0], access['userID'][0]

    def __init__(self,
                 api_key=None,
                 library_id=None,
                 library_type='user',
                 autosync=False):
        """ Service class for communicating with the Zotero API.

        This is mainly a thin wrapper around :py:class:`pyzotero.zotero.Zotero`
        that handles things like transparent HTML<->[edit-formt] conversion.

        :param api_key:     API key for the Zotero API, will be loaded from
                            the configuration if not specified
        :param library_id:  Zotero library ID the API key is valid for, will
                            be loaded from the configuration if not specified
        :param library_type: Type of the library, can be 'user' or 'group'
        """
        self._logger = logging.getLogger()
        idx_path = os.path.join(click.get_app_dir(APP_NAME), 'index.sqlite')
        self.config = load_config()
        self.note_format = self.config['zotcli.note_format']
        self.storage_dir = self.config.get('zotcli.storage_dir')
        self.betterbibtex = self.config.get('zotcli.betterbibtex')
        if self.config.get('zotcli.app_dir'):
            self.app_dir = self.config.get('zotcli.app_dir')

        api_key = api_key or self.config.get('zotcli.api_key')
        library_id = library_id or self.config.get('zotcli.library_id')

        if not api_key or not library_id:
            raise ValueError(
                "Please set your API key and library ID by running "
                "`zotcli configure` or pass them as command-line options.")
        self._zot = Zotero(library_id=library_id,
                           api_key=api_key,
                           library_type=library_type)
        self._index = SearchIndex(idx_path)
        sync_interval = self.config.get('zotcli.sync_interval', 300)
        since_last_sync = int(time.time()) - self._index.last_modified
        if autosync and since_last_sync >= int(sync_interval):
            click.echo("{} seconds since last sync, synchronizing.".format(
                since_last_sync))
            num_updated = self.synchronize()
            click.echo("Updated {} items".format(num_updated))

    def getBetterBibtexKeys(self):
        with open(
                os.path.join(
                    self.app_dir,
                    'better-bibtex/_better-bibtex.json')) as data_file:
            data = json.load(data_file)
        keys = {}
        for i in data['collections'][0]['data']:
            keys[i['itemKey']] = i['citekey']
        return keys

    def synchronize(self):
        """ Update the local index to the latest library version. """
        new_items = tuple(self.items(since=self._index.library_version))
        version = int(self._zot.request.headers.get('last-modified-version'))
        self._index.index(new_items, version)
        return len(new_items)

    def search(self, query, limit=None):
        """ Search the local index for items.

        :param query:   A sqlite FTS4 query
        :param limit:   Maximum number of items to return
        :returns:       Generator that yields matching items.
        """
        return self._index.search(query, limit=limit)

    def items(self, query=None, limit=None, recursive=False, since=0):
        """ Get a list of all items in the library matching the arguments.

        :param query:   Filter items by this query string (targets author and
                        title fields)
        :type query:    str/unicode
        :param limit:   Limit maximum number of returned items
        :type limit:    int
        :param recursive: Include non-toplevel items (attachments, notes, etc)
                          in output
        :type recursive: bool
        :returns:       Generator that yields items
        """
        if self.betterbibtex:
            bbtxkeys = self.getBetterBibtexKeys()

        if limit is None:
            limit = 100
        query_args = {'since': since}
        if query:
            query_args['q'] = query
        if limit:
            query_args['limit'] = limit
        query_fn = self._zot.items if recursive else self._zot.top
        # NOTE: Normally we'd use the makeiter method of Zotero, but it seems
        #       to be broken at the moment, thus we call .follow ourselves
        items = query_fn(**query_args)
        last_url = self._zot.links.get('last')
        if last_url:
            while self._zot.links['self'] != last_url:
                items.extend(self._zot.follow())
        for it in items:
            if self.betterbibtex:
                try:
                    citekey = bbtxkeys[it['data']['key']]
                except:
                    citekey = None
            else:
                matches = CITEKEY_PAT.finditer(it['data'].get('extra', ''))
                citekey = next((m.group(1) for m in matches), None)
            yield Item(key=it['data']['key'],
                       creator=it['meta'].get('creatorSummary'),
                       title=it['data'].get('title', "Untitled"),
                       abstract=it['data'].get('abstractNote'),
                       date=it['data'].get('date'),
                       citekey=citekey)

    def notes(self, item_id):
        """ Get a list of all notes for a given item.

        :param item_id:     ID/key of the item to get notes for
        :returns:           Notes for item
        """
        notes = self._zot.children(item_id, itemType="note")
        for note in notes:
            note['data']['note'] = self._make_note(note)
            yield note

    def attachments(self, item_id):
        """ Get a list of all attachments for a given item.

        If a zotero profile directory is specified in the configuration,
        a resolved local file path will be included, if the file exists.

        :param item_id:     ID/key of the item to get attachments for
        :returns:           Attachments for item
        """
        attachments = self._zot.children(item_id, itemType="attachment")
        if self.storage_dir:
            for att in attachments:
                if not att['data']['linkMode'].startswith("imported"):
                    continue
                fpath = os.path.join(self.storage_dir, att['key'],
                                     att['data']['filename'])
                if not os.path.exists(fpath):
                    continue
                att['data']['path'] = fpath
        return attachments

    def get_attachment_path(self, attachment):
        storage_method = self.config['zotcli.sync_method']
        if storage_method == 'zotfile':
            storage = self.config['zotcli.storage_dir']
            return Path(os.path.join(storage, attachment['data']['title']))

        if not attachment['data']['linkMode'].startswith("imported"):
            raise ValueError(
                "Attachment is not stored on server, cannot download!")
        if storage_method == 'local':
            return Path(attachment['data']['path'])
        out_path = TEMP_DIR / attachment['data']['filename']
        if out_path.exists():
            return out_path
        if storage_method == 'zotero':
            self._zot.dump(attachment['key'], path=unicode(TEMP_DIR))
            return out_path
        elif storage_method == 'webdav':
            user = self.config['zotcli.webdav_user']
            password = self.config['zotcli.webdav_pass']
            location = self.config['zotcli.webdav_path']
            zip_url = "{}/zotero/{}.zip".format(location, attachment['key'])
            resp = requests.get(zip_url, auth=(user, password))
            zf = zipfile.ZipFile(StringIO(resp.content))
            zf.extractall(str(TEMP_DIR))
        return out_path

    def _make_note(self, note_data):
        """ Converts a note from HTML to the configured markup.

        If the note was previously edited with zotcli, the original markup
        will be restored. If it was edited with the Zotero UI, it will be
        converted from the HTML via pandoc.


        :param note_html:       HTML of the note
        :param note_version:    Library version the note was last edited
        :returns:               Dictionary with markup, format and version
        """
        data = None
        note_html = note_data['data']['note']
        note_version = note_data['version']
        if "title=\"b'" in note_html:
            # Fix for badly formatted notes from an earlier version (see #26)
            note_html = re.sub(r'title="b\'(.*?)\'"', r'title="\1"', note_html)
            note_html = note_html.replace("\\n", "")
        blobs = DATA_PAT.findall(note_html)
        # Previously edited with zotcli
        if blobs:
            data = decode_blob(blobs[0])
            if 'version' not in data:
                data['version'] = note_version
            note_html = DATA_PAT.sub("", note_html)
        # Not previously edited with zotcli or updated from the Zotero UI
        if not data or data['version'] < note_version:
            if data and data['version'] < note_version:
                self._logger.info("Note changed on server, reloading markup.")
            note_format = data['format'] if data else self.note_format
            data = {
                'format': note_format,
                'text': pypandoc.convert(note_html, note_format,
                                         format='html'),
                'version': note_version
            }
        return data

    def _make_note_html(self, note_data):
        """ Converts the note's text to HTML and adds a dummy element that
            holds the original markup.

        :param note_data:   dict with text, format and version of the note
        :returns:           Note as HTML
        """
        extra_data = DATA_TMPL.format(
            data=encode_blob(note_data).decode('utf8'))
        html = pypandoc.convert(note_data['text'],
                                'html',
                                format=note_data['format'])
        return html + extra_data

    def create_note(self, item_id, note_text):
        """ Create a new note for a given item.

        :param item_id:     ID/key of the item to create the note for
        :param note_text:   Text of the note
        """
        note = self._zot.item_template('note')
        note_data = {
            'format': self.note_format,
            'text': note_text,
            'version': self._zot.last_modified_version(limit=1) + 2
        }
        note['note'] = self._make_note_html(note_data)
        try:
            self._zot.create_items([note], item_id)
        except Exception as e:
            self._logger.error(e)
            with open("note_backup.txt", "w", encoding='utf-8') as fp:
                fp.write(note_data['text'])
            self._logger.warn(
                "Could not upload note to Zotero. You can find the note "
                "markup in 'note_backup.txt' in the current directory")

    def save_note(self, note):
        """ Update an existing note.

        :param note:        The updated note
        """
        raw_data = note['data']['note']
        raw_data['version'] += 1
        note['data']['note'] = self._make_note_html(raw_data)
        try:
            self._zot.update_item(note)
        except Exception as e:
            self._logger.error(e)
            with open("note_backup.txt", "w", encoding='utf-8') as fp:
                fp.write(raw_data['text'])
            self._logger.warn(
                "Could not upload note to Zotero. You can find the note "
                "markup in 'note_backup.txt' in the current directory")
Exemplo n.º 2
0
class ZoteroBackend(object):
    @staticmethod
    def create_api_key():
        """ Interactively create a new API key via Zotero's OAuth API.

        Requires the user to enter a verification key displayed in the browser.

        :returns:   API key and the user's library ID
        """
        auth = OAuth1Service(
            name='zotero',
            consumer_key=CLIENT_KEY,
            consumer_secret=CLIENT_SECRET,
            request_token_url=REQUEST_TOKEN_URL,
            access_token_url=ACCESS_TOKEN_URL,
            authorize_url=AUTH_URL,
            base_url=BASE_URL)
        token, secret = auth.get_request_token(
            params={'oauth_callback': 'oob'})
        auth_url = auth.get_authorize_url(token)
        auth_url += '&' + urlencode({
            'name': 'zotero-cli',
            'library_access': 1,
            'notes_access': 1,
            'write_access': 1,
            'all_groups': 'read'})
        click.echo("Opening {} in browser, please confirm.".format(auth_url))
        click.launch(auth_url)
        verification = click.prompt("Enter verification code")
        token_resp = auth.get_raw_access_token(
            token, secret, method='POST',
            data={'oauth_verifier': verification})
        if not token_resp:
            logging.debug(token_resp.content)
            click.fail("Error during API key generation.")
        access = urlparse.parse_qs(token_resp.text)
        return access['oauth_token'][0], access['userID'][0]

    def __init__(self, api_key=None, library_id=None, library_type='user',
                 autosync=False):
        """ Service class for communicating with the Zotero API.

        This is mainly a thin wrapper around :py:class:`pyzotero.zotero.Zotero`
        that handles things like transparent HTML<->[edit-formt] conversion.

        :param api_key:     API key for the Zotero API, will be loaded from
                            the configuration if not specified
        :param library_id:  Zotero library ID the API key is valid for, will
                            be loaded from the configuration if not specified
        :param library_type: Type of the library, can be 'user' or 'group'
        """
        self._logger = logging.getLogger()
        idx_path = os.path.join(click.get_app_dir(APP_NAME), 'index.sqlite')
        self.config = load_config()
        self.note_format = self.config['zotcli.note_format']
        self.storage_dir = self.config.get('zotcli.storage_dir')

        api_key = api_key or self.config.get('zotcli.api_key')
        library_id = library_id or self.config.get('zotcli.library_id')

        if not api_key or not library_id:
            raise ValueError(
                "Please set your API key and library ID by running "
                "`zotcli configure` or pass them as command-line options.")
        self._zot = Zotero(library_id=library_id, api_key=api_key,
                           library_type=library_type)
        self._index = SearchIndex(idx_path)
        sync_interval = self.config.get('zotcli.sync_interval', 300)
        since_last_sync = int(time.time()) - self._index.last_modified
        if autosync and since_last_sync >= int(sync_interval):
            click.echo("{} seconds since last sync, synchronizing."
                       .format(since_last_sync))
            num_updated = self.synchronize()
            click.echo("Updated {} items".format(num_updated))

    def synchronize(self):
        """ Update the local index to the latest library version. """
        new_items = tuple(self.items(since=self._index.library_version))
        version = int(self._zot.request.headers.get('last-modified-version'))
        self._index.index(new_items, version)
        return len(new_items)

    def search(self, query, limit=None):
        """ Search the local index for items.

        :param query:   A sqlite FTS4 query
        :param limit:   Maximum number of items to return
        :returns:       Generator that yields matching items.
        """
        return self._index.search(query, limit=limit)

    def items(self, query=None, limit=None, recursive=False, since=0):
        """ Get a list of all items in the library matching the arguments.

        :param query:   Filter items by this query string (targets author and
                        title fields)
        :type query:    str/unicode
        :param limit:   Limit maximum number of returned items
        :type limit:    int
        :param recursive: Include non-toplevel items (attachments, notes, etc)
                          in output
        :type recursive: bool
        :returns:       Generator that yields items
        """
        if limit is None:
            limit = 100
        query_args = {'since': since}
        if query:
            query_args['q'] = query
        if limit:
            query_args['limit'] = limit
        query_fn = self._zot.items if recursive else self._zot.top
        # NOTE: Normally we'd use the makeiter method of Zotero, but it seems
        #       to be broken at the moment, thus we call .follow ourselves
        items = query_fn(**query_args)
        last_url = self._zot.links.get('last')
        if last_url:
            while self._zot.links['self'] != last_url:
                items.extend(self._zot.follow())
        for it in items:
            matches = CITEKEY_PAT.finditer(it['data'].get('extra', ''))
            citekey = next((m.group(1) for m in matches), None)
            yield Item(key=it['data']['key'],
                       creator=it['meta'].get('creatorSummary'),
                       title=it['data'].get('title', "Untitled"),
                       abstract=it['data'].get('abstractNote'),
                       date=it['data'].get('date'),
                       citekey=citekey)

    def notes(self, item_id):
        """ Get a list of all notes for a given item.

        :param item_id:     ID/key of the item to get notes for
        :returns:           Notes for item
        """
        notes = self._zot.children(item_id, itemType="note")
        for note in notes:
            note['data']['note'] = self._make_note(note)
            yield note

    def attachments(self, item_id):
        """ Get a list of all attachments for a given item.

        If a zotero profile directory is specified in the configuration,
        a resolved local file path will be included, if the file exists.

        :param item_id:     ID/key of the item to get attachments for
        :returns:           Attachments for item
        """
        attachments = self._zot.children(item_id, itemType="attachment")
        if self.storage_dir:
            for att in attachments:
                if not att['data']['linkMode'].startswith("imported"):
                    continue
                fpath = os.path.join(self.storage_dir, att['key'],
                                     att['data']['filename'])
                if not os.path.exists(fpath):
                    continue
                att['data']['path'] = fpath
        return attachments

    def get_attachment_path(self, attachment):
        if not attachment['data']['linkMode'].startswith("imported"):
            raise ValueError(
                "Attachment is not stored on server, cannot download!")
        storage_method = self.config['zotcli.sync_method']
        if storage_method == 'local':
            return Path(attachment['data']['path'])
        out_path = TEMP_DIR/attachment['data']['filename']
        if out_path.exists():
            return out_path
        if storage_method == 'zotero':
            self._zot.dump(attachment['key'], path=unicode(TEMP_DIR))
            return out_path
        elif storage_method == 'webdav':
            user = self.config['zotcli.webdav_user']
            password = self.config['zotcli.webdav_pass']
            location = self.config['zotcli.webdav_path']
            zip_url = "{}/zotero/{}.zip".format(
                location, attachment['key'])
            resp = requests.get(zip_url, auth=(user, password))
            zf = zipfile.ZipFile(StringIO(resp.content))
            zf.extractall(str(TEMP_DIR))
        return out_path

    def _make_note(self, note_data):
        """ Converts a note from HTML to the configured markup.

        If the note was previously edited with zotcli, the original markup
        will be restored. If it was edited with the Zotero UI, it will be
        converted from the HTML via pandoc.

        :param note_html:       HTML of the note
        :param note_version:    Library version the note was last edited
        :returns:               Dictionary with markup, format and version
        """
        data = None
        note_html = note_data['data']['note']
        note_version = note_data['version']
        if "title=\"b'" in note_html:
            # Fix for badly formatted notes from an earlier version (see #26)
            note_html = re.sub(r'title="b\'(.*?)\'"', r'title="\1"', note_html)
            note_html = note_html.replace("\\n", "")
        blobs = DATA_PAT.findall(note_html)
        # Previously edited with zotcli
        if blobs:
            data = decode_blob(blobs[0])
            if 'version' not in data:
                data['version'] = note_version
            note_html = DATA_PAT.sub("", note_html)
        # Not previously edited with zotcli or updated from the Zotero UI
        if not data or data['version'] < note_version:
            if data and data['version'] < note_version:
                self._logger.info("Note changed on server, reloading markup.")
            note_format = data['format'] if data else self.note_format
            data = {
                'format': note_format,
                'text': pypandoc.convert(
                    note_html, note_format, format='html'),
                'version': note_version}
        return data

    def _make_note_html(self, note_data):
        """ Converts the note's text to HTML and adds a dummy element that
            holds the original markup.

        :param note_data:   dict with text, format and version of the note
        :returns:           Note as HTML
        """
        extra_data = DATA_TMPL.format(
            data=encode_blob(note_data).decode('utf8'))
        html = pypandoc.convert(note_data['text'], 'html',
                                format=note_data['format'])
        return html + extra_data

    def create_note(self, item_id, note_text):
        """ Create a new note for a given item.

        :param item_id:     ID/key of the item to create the note for
        :param note_text:   Text of the note
        """
        note = self._zot.item_template('note')
        note_data = {'format': self.note_format,
                     'text': note_text,
                     'version': self._zot.last_modified_version(limit=1)+2}
        note['note'] = self._make_note_html(note_data)
        try:
            self._zot.create_items([note], item_id)
        except Exception as e:
            self._logger.error(e)
            with open("note_backup.txt", "w", encoding='utf-8') as fp:
                fp.write(note_data['text'])
            self._logger.warn(
                "Could not upload note to Zotero. You can find the note "
                "markup in 'note_backup.txt' in the current directory")

    def save_note(self, note):
        """ Update an existing note.

        :param note:        The updated note
        """
        raw_data = note['data']['note']
        raw_data['version'] += 1
        note['data']['note'] = self._make_note_html(raw_data)
        try:
            self._zot.update_item(note)
        except Exception as e:
            self._logger.error(e)
            with open("note_backup.txt", "w", encoding='utf-8') as fp:
                fp.write(raw_data['text'])
            self._logger.warn(
                "Could not upload note to Zotero. You can find the note "
                "markup in 'note_backup.txt' in the current directory")