class ZoteroWrap: CACHE_REFERENCE_LIST = "references" CACHE_REFERENCE_TYPES = "reference_types" CACHE_REFERENCE_TEMPLATES = "reference_templates" def __init__(self, library_id, library_type, api_key, directory): cache_filename = "{}-{}-{}.pkl".format(library_id, library_type, api_key) self.cache_path = os.path.join(directory, cache_filename) # reference_types and reference_templates must have the same ordering. self.reference_types = [] self.reference_templates = {} self._zotero_lib = Zotero(library_id, library_type, api_key) self._references = [] # Data I/O methods section. def initialize(self): """Load the cached Zotero data, or retrieve them if there is none.""" try: self.load_cache() except FileNotFoundError: self.load_distant() def load_cache(self): """Load the cached Zotero data.""" with open(self.cache_path, "rb") as f: print("Loading cached Zotero data...") cache = pickle.load(f) self._references = cache[self.CACHE_REFERENCE_LIST] self.reference_types = cache[self.CACHE_REFERENCE_TYPES] self.reference_templates = cache[self.CACHE_REFERENCE_TEMPLATES] print("Cached Zotero data loaded.") def load_distant(self): """Load the distant Zotero data.""" print("Loading distant Zotero data...") self._references = self.get_references() self.reference_types = self.get_reference_types() self.reference_templates = self.get_reference_templates( self.reference_types) print("Distant Zotero data loaded.") self.cache() def cache(self): """Cache the Zotero data.""" with open(self.cache_path, "wb") as f: cache = { self.CACHE_REFERENCE_LIST: self._references, self.CACHE_REFERENCE_TYPES: self.reference_types, self.CACHE_REFERENCE_TEMPLATES: self.reference_templates } pickle.dump(cache, f) def create_local_reference(self, ref): """Append the reference at the end of the reference list and cache it.""" self._references.append(ref) self.cache() def create_distant_reference(self, ref_data): """Validate and create the reference in Zotero and return the created item.""" self.validate_reference_data(ref_data) creation_status = self._zotero_lib.create_items([ref_data]) try: created_item = creation_status["successful"]["0"] return created_item except KeyError as e: print(creation_status) raise CreateZoteroItemError from e def update_local_reference(self, index, ref): """Replace the reference in the reference list and cache it.""" self._references[index] = ref self.cache() def update_distant_reference(self, ref): """Validate and update the reference in Zotero. Existing fields not present will be left unmodified. """ self.validate_reference_data(ref["data"]) self._zotero_lib.update_item(ref) def validate_reference_data(self, ref_data): """Validate the reference data. Zotero.check_items() caches data after the first API call. """ try: self._zotero_lib.check_items([ref_data]) except InvalidItemFields as e: raise InvalidZoteroItemError from e def get_references(self): """Return all references in the Zotero database. Takes time...""" return self._zotero_lib.everything(self._zotero_lib.top()) def get_reference_types(self): """Return the reference types. Zotero.item_types() caches data after the first API call. """ item_types = self._zotero_lib.item_types() return sorted([x["itemType"] for x in item_types]) def get_reference_templates(self, ref_types): """Return the reference templates for the types as an ordered dictionary.""" return OrderedDict([(x, self.get_reference_template(x)) for x in ref_types]) def get_reference_template(self, ref_type): """Return the reference template for the type as an ordered dictionary. Zotero.item_template() caches data after the first API call. """ template = self._zotero_lib.item_template(ref_type) return OrderedDict(sorted(template.items(), key=lambda x: x[0])) def get_reference(self, ref_key): """Return the reference for the key.""" return self._zotero_lib.item(ref_key) # Public @properties surrogates section. def reference_count(self): """Return the number of references.""" return len(self._references) def reference_data(self, index): """Return the 'data' field of the reference.""" return self._references[index]["data"] def reference_extra_field(self, field, index): """Return the value of the field in 'extra', otherwise ''.""" ref_data = self.reference_data(index) extra_fields = ref_data["extra"].split("\n") field_id = field + ":" matched = next((x for x in extra_fields if x.startswith(field_id)), None) if matched: return matched.replace(field_id, "", 1).strip() else: return "" def reference_type(self, index): """Return the reference type.""" return self.reference_data(index)["itemType"] def reference_key(self, index): """Return the reference key.""" return self._references[index]["key"] def reference_id(self, index): """Return the reference ID (locally defined).""" # TODO Include ISBN and ISSN? doi = self.reference_doi(index) if doi: return doi else: pmid = self.reference_pmid(index) if pmid: return "PMID_" + pmid else: unpublished_id = self.reference_unpublished_id(index) if unpublished_id: return "UNPUBLISHED_" + unpublished_id return "" def reference_doi(self, index): """Return the reference DOI.""" return self.reference_data(index).get( "DOI", self.reference_extra_field("DOI", index)) def reference_pmid(self, index): """Return the reference PMID.""" return self.reference_extra_field("PMID", index) def reference_unpublished_id(self, index): """Return the reference UNPUBLISHED ID.""" return self.reference_extra_field("UNPUBLISHED", index) def reference_title(self, index): """Return the reference title.""" return self.reference_data(index)["title"] def reference_creator_surnames(self, index): """Return as a list the surnames of the reference creators (locally defined).""" # TODO Not true, ex: ISBN 978-1-4398-3778-8. Return all creator types? # Academic books published as a collection of chapters contributed by # different authors have editors but not authors at the level of the # book (as opposed to the level of a chapter). creators = self.reference_data(index)["creators"] creator_types = [x["creatorType"] for x in creators] # 'name' (not split) might be used instead of 'firstName' and 'lastName'. try: if "author" in creator_types: return [ x["lastName"] for x in creators if x["creatorType"] == "author" ] else: return [x["lastName"] for x in creators] except KeyError: return [] def reference_creator_surnames_str(self, index): """Return as a string the surnames of the reference creators (locally defined).""" # NB: str.join() returns an empty string for an empty list. return ", ".join(self.reference_creator_surnames(index)) def reference_date(self, index): """Return the reference publication date.""" return self.reference_data(index)["date"] def reference_year(self, index): """Return the reference publication year.""" # TODO Use meta:parsedDate field instead? ref_date = self.reference_date(index) try: # NB: datetime.year returns an int. return parse(ref_date).year except ValueError: matched = re.search(r"\d{4}", ref_date) if matched: return int(matched.group()) else: return "" def reference_journal(self, index): """Return the reference journal name.""" # TODO Change the column name 'Journal' to an other? ref_type = self.reference_type(index) if ref_type == "journalArticle": return self.reference_data(index)["publicationTitle"] else: return "({})".format(ref_type) # Public methods section. def reference_index(self, ref_id): """Return the first reference with this ID.""" try: indexes = range(self.reference_count()) return next(i for i in indexes if self.reference_id(i) == ref_id) except StopIteration as e: raise ReferenceNotFoundError("ID: " + ref_id) from e def reference_creators_citation(self, ref_id): """Return for citation the creator surnames (locally defined) and the publication year.""" # FIXME Delayed refactoring. Use an index instead of an ID. index = self.reference_index(ref_id) creators = self.reference_creator_surnames(index) creator_count = len(creators) if creator_count == 0: return "" year = self.reference_year(index) if creator_count == 1: return "{} ({})".format(creators[0], year) elif creator_count == 2: return "{} and {} ({})".format(creators[0], creators[1], year) else: return "{} et al. ({})".format(creators[0], year)
def _from_zotero_library(engine, library_id, library_type, api_key=None, verbose=False): library_type_id = "zot_%s_%i" % (library_type[:1], library_id) # Every library gets a separate schema within the database item_type_schema = schema.for_library(engine, library_type_id, verbose) # returns dictionary of item table fields. # Setup the Zotero connection through pyzotero z = Zotero(library_id, library_type, api_key) check_access = z.items(limit=1, format="json", includeTrashed=1) library_name = check_access[0]['library']['name'] print("\n%s %s ¶" % (library_type_id, library_name)) # Start the engine and fetch items from the cloud! with engine.connect() as db: # Start sync timer and log attempt to sync. # Duration and latest version will be updated when finished. query = """ INSERT INTO logs.zot_fetch (timestamp, library, name) VALUES ( DEFAULT, :lib, :name) RETURNING id,timestamp; """ sync = db.execute(text(query), lib=library_type_id, name=library_name).fetchone() # ( Int, datetime ) print("Sync #%i was started at %s" % (sync[0], sync[1].strftime('%c'))) # Get current local library version query = """ SELECT version FROM logs.zot_fetch WHERE library='%s' AND duration IS NOT NULL ORDER BY timestamp DESC LIMIT 1; """ % library_type_id res_last_sync_version = db.execute( text(query)).fetchone() # ( Int, ) or None if res_last_sync_version: last_sync_version = res_last_sync_version[0] query = """ SELECT COUNT(*) FROM %s.items WHERE NOT deleted ; """ % library_type_id local_count = db.execute( text(query)).fetchone() # ( Int, ) or None print("local mirror is at version %i and contains %i items" % (last_sync_version, local_count[0])) else: last_sync_version = 0 print("Starting initial sync of library %s" % library_type_id) # Get current remote library count and version z.top(limit=1, format='keys') remote_count = int(z.request.headers.get('total-results', 0)) library_version = int(z.request.headers.get('last-modified-version', 0)) print("remote cloud is at version %i and contains %i items" % (library_version, remote_count)) if last_sync_version < library_version: # Get list of local item keys and their versions query = """ SELECT key,version FROM %s.items ; """ % library_type_id local_versions = dict(db.execute( text(query)).fetchall()) # { String: Int, } def _fetch_updates_and_inserts(start=0): start_round = _start_duration() inserts = 0 update_list = z.top(limit=100, start=start, format='json', since=last_sync_version, includeTrashed=1) total_results = int(z.request.headers.get('Total-Results')) # Maybe there are only deletions to handle, so checking number of updates to handle if len(update_list) > 0: for item in update_list: data = {} for field, value in item['data'].items(): data[field] = schema._typeset_for_db( field, value, item['data']['itemType']) if field == 'version': update_string = '"version"=:version' insert_field_string = '"key", "version"' insert_value_string = ':key, :version' elif field != 'key': update_string += ', "%s"=:%s' % (field, field) insert_field_string += ', "%s"' % field insert_value_string += ', :%s' % field if field == 'note': data['customJSON'] = schema._typeset_for_db( "customJSON", json.loads(re.findall(r'{.*}', value)[0]), "note") update_string += ', "%s"=:%s' % ("customJSON", "customJSON") insert_field_string += ', "%s"' % "customJSON" insert_value_string += ', :%s' % "customJSON" for field, value in item['meta'].items(): data[field] = schema._typeset_for_db( field, value, item['data']['itemType']) update_string += ', "%s"=:%s' % (field, field) insert_field_string += ', "%s"' % field insert_value_string += ', :%s' % field item_type = item['data']['itemType'] if item['key'] in local_versions: query = """ UPDATE %s."%s" SET %s WHERE key=:key ; """ % (library_type_id, item_type, update_string) db.execute(text(query), **data) else: query = """ INSERT INTO %s."%s" (%s) VALUES ( %s ) ; """ % (library_type_id, item_type, insert_field_string, insert_value_string) db.execute(text(query), **data) inserts += 1 round_duration = _duration(start_round) print("Finished processing %i updates in %s seconds." % (len(update_list), str(round_duration))) if len(update_list) == 100 and start + 100 < total_results: print( "%i of %i updates done: fetching more updates now." % (start + 100, total_results)) inserts = inserts + _fetch_updates_and_inserts( start=start + 100) else: print("%i of %i updates have been processed." % (total_results, total_results)) else: round_duration = _duration(start_round) print( "Zero updates to process (it took %s seconds to figure that out)" % str(round_duration)) return inserts # fetch all updates in batches of 100 (includes updates to existing items and new items) inserts = _fetch_updates_and_inserts() def _fetch_deletions(since_version): deletions = 0 start_round = _start_duration() print("Fetching list of deletions since last successful sync.") # Get list of deleted items from cloud delete_list = z.deleted(since=since_version) if len(delete_list['items']) > 0: for item in delete_list['items']: if item in local_versions: query = """ DELETE FROM %s.items WHERE key=:key ; """ % library_type_id db.execute(text(query), key=item) deletions += 1 else: print( "Tried to DELETE item with key %s, but this item is not in local library..." % item) round_duration = _duration(start_round) print("Finished processing %i deletions in %s seconds" % (len(delete_list['items']), str(round_duration))) return deletions # if this is not the initial sync, there's nothing to delete... if last_sync_version > 0: deletions = _fetch_deletions(last_sync_version) final_count = local_count[0] + inserts - deletions else: print( "Initial sync has been successful. Next time atomic updates will be performed!" ) else: print("Nothing to sync, everything is up to date.") duration = _duration(sync[1]) query = """ UPDATE logs.zot_fetch SET duration=:duration, version=:version WHERE id=:id ; """ db.execute(text(query), duration=math.ceil(duration), version=library_version, id=sync[0]) # Closing connection to database ༺ with engine.connect() as db : ༻ print("Syncing library %s took %s seconds\n" % (library_type_id, str(duration)))
class ZoteroWrap: CACHE_REFERENCE_LIST = "references" CACHE_REFERENCE_TYPES = "reference_types" CACHE_REFERENCE_TEMPLATES = "reference_templates" def __init__(self, library_id, library_type, api_key, directory): cache_filename = "{}-{}-{}.pkl".format(library_id, library_type, api_key) self.cache_path = os.path.join(directory, cache_filename) # reference_types and reference_templates must have the same ordering. self.reference_types = [] self.reference_templates = {} self._zotero_lib = Zotero(library_id, library_type, api_key) self._references = [] # Data I/O methods section. def initialize(self): """Load the cached Zotero data, or retrieve them if there is none.""" try: self.load_cache() except FileNotFoundError: self.load_distant() def load_cache(self): """Load the cached Zotero data.""" with open(self.cache_path, "rb") as f: print("Loading cached Zotero data...") cache = pickle.load(f) self._references = cache[self.CACHE_REFERENCE_LIST] self.reference_types = cache[self.CACHE_REFERENCE_TYPES] self.reference_templates = cache[self.CACHE_REFERENCE_TEMPLATES] print("Cached Zotero data loaded.") def load_distant(self): """Load the distant Zotero data.""" print("Loading distant Zotero data...") self._references = self.get_references() self.reference_types = self.get_reference_types() self.reference_templates = self.get_reference_templates(self.reference_types) print("Distant Zotero data loaded.") self.cache() def cache(self): """Cache the Zotero data.""" with open(self.cache_path, "wb") as f: cache = {self.CACHE_REFERENCE_LIST: self._references, self.CACHE_REFERENCE_TYPES: self.reference_types, self.CACHE_REFERENCE_TEMPLATES: self.reference_templates} pickle.dump(cache, f) def create_local_reference(self, ref): """Append the reference at the end of the reference list and cache it.""" self._references.append(ref) self.cache() def create_distant_reference(self, ref_data): """Validate and create the reference in Zotero and return the created item.""" self.validate_reference_data(ref_data) creation_status = self._zotero_lib.create_items([ref_data]) try: created_item = creation_status["successful"]["0"] return created_item except KeyError as e: print(creation_status) raise CreateZoteroItemError from e def update_local_reference(self, index, ref): """Replace the reference in the reference list and cache it.""" self._references[index] = ref self.cache() def update_distant_reference(self, ref): """Validate and update the reference in Zotero. Existing fields not present will be left unmodified. """ self.validate_reference_data(ref["data"]) self._zotero_lib.update_item(ref) def validate_reference_data(self, ref_data): """Validate the reference data. Zotero.check_items() caches data after the first API call. """ try: self._zotero_lib.check_items([ref_data]) except InvalidItemFields as e: raise InvalidZoteroItemError from e def get_references(self): """Return all references in the Zotero database. Takes time...""" return self._zotero_lib.everything(self._zotero_lib.top()) def get_reference_types(self): """Return the reference types. Zotero.item_types() caches data after the first API call. """ item_types = self._zotero_lib.item_types() return sorted([x["itemType"] for x in item_types]) def get_reference_templates(self, ref_types): """Return the reference templates for the types as an ordered dictionary.""" return OrderedDict([(x, self.get_reference_template(x)) for x in ref_types]) def get_reference_template(self, ref_type): """Return the reference template for the type as an ordered dictionary. Zotero.item_template() caches data after the first API call. """ template = self._zotero_lib.item_template(ref_type) return OrderedDict(sorted(template.items(), key=lambda x: x[0])) def get_reference(self, ref_key): """Return the reference for the key.""" return self._zotero_lib.item(ref_key) # Public @properties surrogates section. def reference_count(self): """Return the number of references.""" return len(self._references) def reference_data(self, index): """Return the 'data' field of the reference.""" return self._references[index]["data"] def reference_extra_field(self, field, index): """Return the value of the field in 'extra', otherwise ''.""" ref_data = self.reference_data(index) extra_fields = ref_data["extra"].split("\n") field_id = field + ":" matched = next((x for x in extra_fields if x.startswith(field_id)), None) if matched: return matched.replace(field_id, "", 1).strip() else: return "" def reference_type(self, index): """Return the reference type.""" return self.reference_data(index)["itemType"] def reference_key(self, index): """Return the reference key.""" return self._references[index]["key"] def reference_id(self, index): """Return the reference ID (locally defined).""" # TODO Include ISBN and ISSN? doi = self.reference_doi(index) if doi: return doi else: pmid = self.reference_pmid(index) if pmid: return "PMID_" + pmid else: unpublished_id = self.reference_unpublished_id(index) if unpublished_id: return "UNPUBLISHED_" + unpublished_id return "" def reference_doi(self, index): """Return the reference DOI.""" return self.reference_data(index).get("DOI", self.reference_extra_field("DOI", index)) def reference_pmid(self, index): """Return the reference PMID.""" return self.reference_extra_field("PMID", index) def reference_unpublished_id(self, index): """Return the reference UNPUBLISHED ID.""" return self.reference_extra_field("UNPUBLISHED", index) def reference_title(self, index): """Return the reference title.""" return self.reference_data(index)["title"] def reference_creator_surnames(self, index): """Return as a list the surnames of the reference creators (locally defined).""" # TODO Not true, ex: ISBN 978-1-4398-3778-8. Return all creator types? # Academic books published as a collection of chapters contributed by # different authors have editors but not authors at the level of the # book (as opposed to the level of a chapter). creators = self.reference_data(index)["creators"] creator_types = [x["creatorType"] for x in creators] # 'name' (not split) might be used instead of 'firstName' and 'lastName'. try: if "author" in creator_types: return [x["lastName"] for x in creators if x["creatorType"] == "author"] else: return [x["lastName"] for x in creators] except KeyError: return [] def reference_creator_surnames_str(self, index): """Return as a string the surnames of the reference creators (locally defined).""" # NB: str.join() returns an empty string for an empty list. return ", ".join(self.reference_creator_surnames(index)) def reference_date(self, index): """Return the reference publication date.""" return self.reference_data(index)["date"] def reference_year(self, index): """Return the reference publication year.""" # TODO Use meta:parsedDate field instead? ref_date = self.reference_date(index) try: # NB: datetime.year returns an int. return parse(ref_date).year except ValueError: matched = re.search(r"\d{4}", ref_date) if matched: return int(matched.group()) else: return "" def reference_journal(self, index): """Return the reference journal name.""" # TODO Change the column name 'Journal' to an other? ref_type = self.reference_type(index) if ref_type == "journalArticle": return self.reference_data(index)["publicationTitle"] else: return "({})".format(ref_type) # Public methods section. def reference_index(self, ref_id): """Return the first reference with this ID.""" try: indexes = range(self.reference_count()) return next(i for i in indexes if self.reference_id(i) == ref_id) except StopIteration as e: raise ReferenceNotFoundError("ID: " + ref_id) from e def reference_creators_citation(self, ref_id): """Return for citation the creator surnames (locally defined) and the publication year.""" # FIXME Delayed refactoring. Use an index instead of an ID. index = self.reference_index(ref_id) creators = self.reference_creator_surnames(index) creator_count = len(creators) if creator_count == 0: return "" year = self.reference_year(index) if creator_count == 1: return "{} ({})".format(creators[0], year) elif creator_count == 2: return "{} and {} ({})".format(creators[0], creators[1], year) else: return "{} et al. ({})".format(creators[0], year)