def __init__(self, api_key=None, library_id=None, library_type='user'): """ Service class for communicating with the Zotero API. This is mainly a thin wrapper around :py:class:`pyzotero.zotero.Zotero` that handles things like transparent HTML<->[edit-formt] conversion. :param api_key: API key for the Zotero API, will be loaded from the configuration if not specified :param library_id: Zotero library ID the API key is valid for, will be loaded from the configuration if not specified :param library_type: Type of the library, can be 'user' or 'group' """ self._logger = logging.getLogger() idx_path = os.path.join(click.get_app_dir(APP_NAME), 'index.sqlite') self.config = load_config() self.note_format = self.config['zotcli.note_format'] self.storage_dir = self.config.get('zotcli.storage_dir') api_key = api_key or self.config.get('zotcli.api_key') library_id = library_id or self.config.get('zotcli.library_id') if not api_key or not library_id: raise ValueError( "Please set your API key and library ID by running " "`zotcli configure` or pass them as command-line options.") self._zot = Zotero(library_id=library_id, api_key=api_key, library_type=library_type) self._index = SearchIndex(idx_path) sync_interval = self.config.get('zotcli.sync_interval', 300) since_last_sync = int(time.time()) - self._index.last_modified if since_last_sync >= int(sync_interval): self._logger.info("{} seconds since last sync, synchronizing." .format(since_last_sync)) self.synchronize()
def _from_all_groups_by_user(engine, user_id=None, api_key=None, only_groups=False, verbose=False): z = Zotero(user_id, 'user', api_key) groups = z.groups() for group in groups: from_zotero_group(engine, group['id'], api_key, verbose)
def __init__(self, library_id, library_type, api_key, directory): cache_filename = "{}-{}-{}.pkl".format(library_id, library_type, api_key) self.cache_path = os.path.join(directory, cache_filename) # reference_types and reference_templates must have the same ordering. self.reference_types = [] self.reference_templates = {} self._zotero_lib = Zotero(library_id, library_type, api_key) self._references = []
def connect(library=None, api_key=None): config_library = get_library() if library is None: library = config_library else: library = str(library) # user input if library is None: while True: library = raw_input("Library ID: ") if library and library.isdigit(): break print("Library needs to be a sequence of digits, not %r" % library) if api_key is None: api_key = keyring.get_password(KEYRING_DOMAIN, library) api_key_changed = False else: api_key_changed = True msg_printed = False while True: if not api_key: if not msg_printed: print("Please enter the library API key " "(see https://www.zotero.org/settings/keys/new)") msg_printed = True api_key = raw_input("Library API key (ctrl-c to abort): ") z = Zotero(library, 'user', api_key, True) try: z.num_items() except UserNotAuthorised: print("Connection refused, invalid API key...") api_key = None else: # store new configuration if library != config_library: set_library(library) if api_key_changed: keyring.set_password(KEYRING_DOMAIN, library, api_key) return z
def __init__(self, library_id, library_type, api_key, papers2, keyword_types=('user', 'label'), label_map={}, add_to_collections=[], upload_attachments="all", batch_size=50, checkpoint=None, dryrun=None): self.client = Zotero(library_id, library_type, api_key) self.papers2 = papers2 self.keyword_types = keyword_types self.label_map = label_map self.upload_attachments = upload_attachments self.checkpoint = checkpoint self.dryrun = JSONWriter(dryrun) if dryrun is not None else None self._batch = Batch(batch_size) self._load_collections(add_to_collections)
def _from_zotero_library(engine, library_id, library_type, api_key=None, verbose=False): library_type_id = "zot_%s_%i" % (library_type[:1], library_id) # Every library gets a separate schema within the database item_type_schema = schema.for_library(engine, library_type_id, verbose) # returns dictionary of item table fields. # Setup the Zotero connection through pyzotero z = Zotero(library_id, library_type, api_key) check_access = z.items(limit=1, format="json", includeTrashed=1) library_name = check_access[0]['library']['name'] print("\n%s %s ¶" % (library_type_id, library_name)) # Start the engine and fetch items from the cloud! with engine.connect() as db: # Start sync timer and log attempt to sync. # Duration and latest version will be updated when finished. query = """ INSERT INTO logs.zot_fetch (timestamp, library, name) VALUES ( DEFAULT, :lib, :name) RETURNING id,timestamp; """ sync = db.execute(text(query), lib=library_type_id, name=library_name).fetchone() # ( Int, datetime ) print("Sync #%i was started at %s" % (sync[0], sync[1].strftime('%c'))) # Get current local library version query = """ SELECT version FROM logs.zot_fetch WHERE library='%s' AND duration IS NOT NULL ORDER BY timestamp DESC LIMIT 1; """ % library_type_id res_last_sync_version = db.execute( text(query)).fetchone() # ( Int, ) or None if res_last_sync_version: last_sync_version = res_last_sync_version[0] query = """ SELECT COUNT(*) FROM %s.items WHERE NOT deleted ; """ % library_type_id local_count = db.execute( text(query)).fetchone() # ( Int, ) or None print("local mirror is at version %i and contains %i items" % (last_sync_version, local_count[0])) else: last_sync_version = 0 print("Starting initial sync of library %s" % library_type_id) # Get current remote library count and version z.top(limit=1, format='keys') remote_count = int(z.request.headers.get('total-results', 0)) library_version = int(z.request.headers.get('last-modified-version', 0)) print("remote cloud is at version %i and contains %i items" % (library_version, remote_count)) if last_sync_version < library_version: # Get list of local item keys and their versions query = """ SELECT key,version FROM %s.items ; """ % library_type_id local_versions = dict(db.execute( text(query)).fetchall()) # { String: Int, } def _fetch_updates_and_inserts(start=0): start_round = _start_duration() inserts = 0 update_list = z.top(limit=100, start=start, format='json', since=last_sync_version, includeTrashed=1) total_results = int(z.request.headers.get('Total-Results')) # Maybe there are only deletions to handle, so checking number of updates to handle if len(update_list) > 0: for item in update_list: data = {} for field, value in item['data'].items(): data[field] = schema._typeset_for_db( field, value, item['data']['itemType']) if field == 'version': update_string = '"version"=:version' insert_field_string = '"key", "version"' insert_value_string = ':key, :version' elif field != 'key': update_string += ', "%s"=:%s' % (field, field) insert_field_string += ', "%s"' % field insert_value_string += ', :%s' % field if field == 'note': data['customJSON'] = schema._typeset_for_db( "customJSON", json.loads(re.findall(r'{.*}', value)[0]), "note") update_string += ', "%s"=:%s' % ("customJSON", "customJSON") insert_field_string += ', "%s"' % "customJSON" insert_value_string += ', :%s' % "customJSON" for field, value in item['meta'].items(): data[field] = schema._typeset_for_db( field, value, item['data']['itemType']) update_string += ', "%s"=:%s' % (field, field) insert_field_string += ', "%s"' % field insert_value_string += ', :%s' % field item_type = item['data']['itemType'] if item['key'] in local_versions: query = """ UPDATE %s."%s" SET %s WHERE key=:key ; """ % (library_type_id, item_type, update_string) db.execute(text(query), **data) else: query = """ INSERT INTO %s."%s" (%s) VALUES ( %s ) ; """ % (library_type_id, item_type, insert_field_string, insert_value_string) db.execute(text(query), **data) inserts += 1 round_duration = _duration(start_round) print("Finished processing %i updates in %s seconds." % (len(update_list), str(round_duration))) if len(update_list) == 100 and start + 100 < total_results: print( "%i of %i updates done: fetching more updates now." % (start + 100, total_results)) inserts = inserts + _fetch_updates_and_inserts( start=start + 100) else: print("%i of %i updates have been processed." % (total_results, total_results)) else: round_duration = _duration(start_round) print( "Zero updates to process (it took %s seconds to figure that out)" % str(round_duration)) return inserts # fetch all updates in batches of 100 (includes updates to existing items and new items) inserts = _fetch_updates_and_inserts() def _fetch_deletions(since_version): deletions = 0 start_round = _start_duration() print("Fetching list of deletions since last successful sync.") # Get list of deleted items from cloud delete_list = z.deleted(since=since_version) if len(delete_list['items']) > 0: for item in delete_list['items']: if item in local_versions: query = """ DELETE FROM %s.items WHERE key=:key ; """ % library_type_id db.execute(text(query), key=item) deletions += 1 else: print( "Tried to DELETE item with key %s, but this item is not in local library..." % item) round_duration = _duration(start_round) print("Finished processing %i deletions in %s seconds" % (len(delete_list['items']), str(round_duration))) return deletions # if this is not the initial sync, there's nothing to delete... if last_sync_version > 0: deletions = _fetch_deletions(last_sync_version) final_count = local_count[0] + inserts - deletions else: print( "Initial sync has been successful. Next time atomic updates will be performed!" ) else: print("Nothing to sync, everything is up to date.") duration = _duration(sync[1]) query = """ UPDATE logs.zot_fetch SET duration=:duration, version=:version WHERE id=:id ; """ db.execute(text(query), duration=math.ceil(duration), version=library_version, id=sync[0]) # Closing connection to database ༺ with engine.connect() as db : ༻ print("Syncing library %s took %s seconds\n" % (library_type_id, str(duration)))
def key_info(api_key, verbose=False): print('checking API key properties...') z = Zotero('AnyLibrary', 'AnyType', api_key) return z.key_info(limit=None)