def test_isbn_needs_normalization(): # TODO: Add some isbn's that need normalization
for isbn in "0 1234567890 Bob".split():
assert notisbn(isbn)
assert not NormalizeISBNJob.isbn_needs_normalization(isbn)
for isbn in "0123456789 0425016013 9780425016015 0441788386 9780441788385".split():
assert not notisbn(isbn) # double negative
assert not NormalizeISBNJob.isbn_needs_normalization(isbn)
for isbn in ("ISBN 13: 9780425016015", "(ebook)0441788386"):
assert not NormalizeISBNJob.isbn_needs_normalization(isbn)
def is_valid(isbn):
"""Check whether an ISBN 10 or 13 is valid."""
# isbnlib marks any ISBN10 with lowercase 'X' as invalid
isbn = isbn.upper()
try:
stdnum.isbn
except NameError:
pass
else:
try:
stdnum.isbn.validate(isbn)
except stdnum.isbn.InvalidFormat as e:
raise InvalidIsbnException(str(e))
except stdnum.isbn.InvalidChecksum as e:
raise InvalidIsbnException(str(e))
except stdnum.isbn.InvalidLength as e:
raise InvalidIsbnException(str(e))
return True
try:
isbnlib
except NameError:
pass
else:
if isbnlib.notisbn(isbn):
raise InvalidIsbnException('Invalid ISBN found')
return True
getIsbn(isbn)
return True
async def check(self, entry):
length = self._cfg.get('isbn_length', entry, 13)
if not length:
return []
isbn = entry.data.get('isbn')
if not isbn:
return []
clean_isbn = clean(isbn)
if not clean_isbn or notisbn(clean_isbn):
return []
if length not in (10, 13):
raise ConfigurationError(
"The option 'isbn_length' must be either of 10 or 13.")
if length == 10:
if not is_isbn10(clean_isbn):
return [(type(self).NAME,
"ISBN '{}' is not of length 10.".format(isbn),
"ISBN-10 would be '{}'".format(to_isbn10(clean_isbn)))
]
elif length == 13:
if not is_isbn13(clean_isbn):
return [(type(self).NAME,
"ISBN '{}' is not of length 13.".format(isbn),
"ISBN-13 would be '{}'".format(to_isbn13(clean_isbn)))
]
return []
async def check(self, entry):
fmt = self._cfg.get('isbn_format', entry)
if not fmt:
return []
isbn = entry.data.get('isbn')
if not isbn:
return []
clean_isbn = clean(isbn)
if not clean_isbn or notisbn(clean_isbn):
return []
if fmt not in ('canonical', 'masked'):
raise ConfigurationError("The option 'isbn_format' must be \
either of 'canonical' or 'masked'.")
if fmt == 'canonical':
cisbn = canonical(clean_isbn)
if cisbn != isbn:
return [(type(self).NAME,
"ISBN '{}' is not in canonical format.".format(isbn),
"Canonical format would be '{}'".format(cisbn))]
elif fmt == 'masked':
misbn = mask(clean_isbn)
if misbn != isbn:
return [(type(self).NAME,
"ISBN '{}' is not in masked format.".format(isbn),
"Masked format would be '{}'".format(misbn))]
return []
def isbn(self,isbn): #adds isbn to google spread sheet #check if valid clean_isbn = isbnlib.clean(isbn) if isbnlib.notisbn(clean_isbn): return "not valid isbn" #should check if has been collected before canonical = None; #first check trove canonical = self.trove.extract(clean_isbn); if not canonical : # try alternative isbn form print "trying alternative form " alt_isbn = clean_isbn; if isbnlib.is_isbn13(clean_isbn): alt_isbn = isbnlib.to_isbn10(clean_isbn) else : alt_isbn = isbnlib.to_isbn13(clean_isbn) canonical = self.trove.extract(alt_isbn); if canonical : clean_isbn = alt_isbn if not canonical : canonical = self.__reduce_metadata(clean_isbn,['merge','isbndb','openl']) if not canonical: return "no metadata found for isbn: " + clean_isbn canonical['source']='isbnlib' canonical["Authors"] = u', '.join(canonical["Authors"]) canonical['link']=None row_data = ['isbn:'+clean_isbn, canonical["Title"], canonical["Authors"], canonical["Year"], canonical["Publisher"],canonical['link']] return self.__add_and_render(row_data)
def set_isbn13():
"""If isbn13 is empty, calculate it from isbn10
Remember there can be multiple isbns in the isbn_10 field, due to errors in
the openlibrary.org db. Thus we must split on ';' and calculate isbn13 for
all of the them"""
sql = "select id, isbn_10 from book where isbn_13 = ? and isbn_10 != ?;"
key = ("", "")
cur = conn.cursor()
cur.execute(sql, key)
res = cur.fetchall()
data = []
for book in res:
id, isbns = book
isbns = isbns.split("; ")
isbn_13 = []
for isbn_10 in isbns:
if notisbn(isbn_10):
print(f"error, id {id}, isbn_10 {isbn_10} is not a valid isbn")
continue
isbn_13.append(to_isbn13(isbn_10))
data.append(("; ".join(isbn_13), id))
sql = "UPDATE book SET isbn_13 = ? WHERE id = ?;"
cur.executemany(sql, data)
conn.commit()
def is_valid(isbn):
"""Check whether an ISBN 10 or 13 is valid."""
# isbnlib marks any ISBN10 with lowercase 'X' as invalid
isbn = isbn.upper()
try:
stdnum.isbn
except NameError:
pass
else:
try:
stdnum.isbn.validate(isbn)
except (stdnum.isbn.InvalidFormat, stdnum.isbn.InvalidChecksum,
stdnum.isbn.InvalidLength) as e:
raise InvalidIsbnException(str(e))
return True
try:
isbnlib
except NameError:
pass
else:
if isbnlib.notisbn(isbn):
raise InvalidIsbnException('Invalid ISBN found')
return True
raise NotImplementedError(
'ISBN functionality not available. Install stdnum package.')
def is_valid(isbn):
"""Check whether an ISBN 10 or 13 is valid."""
# isbnlib marks any ISBN10 with lowercase 'X' as invalid
isbn = isbn.upper()
try:
stdnum.isbn
except NameError:
pass
else:
try:
stdnum.isbn.validate(isbn)
except stdnum.isbn.InvalidFormat as e:
raise InvalidIsbnException(str(e))
except stdnum.isbn.InvalidChecksum as e:
raise InvalidIsbnException(str(e))
except stdnum.isbn.InvalidLength as e:
raise InvalidIsbnException(str(e))
return True
try:
isbnlib
except NameError:
pass
else:
if isbnlib.notisbn(isbn):
raise InvalidIsbnException('Invalid ISBN found')
return True
getIsbn(isbn)
return True
def is_valid(isbn_id):
"""
Check that a given string is a valid ISBN.
:param isbn_id: the isbn to be checked.
:returns: boolean indicating whether the isbn is valid or not.
>>> is_valid("978-3-16-148410-0")
True
>>> is_valid("9783161484100")
True
>>> is_valid("9783161484100aa")
False
>>> is_valid("abcd")
False
>>> is_valid("0136091814")
True
>>> is_valid("0136091812")
False
>>> is_valid("9780136091817")
False
>>> is_valid("123456789X")
True
"""
return ((not isbnlib.notisbn(isbn_id)) and
(isbnlib.get_canonical_isbn(isbn_id) == isbn_id
or isbnlib.mask(isbnlib.get_canonical_isbn(isbn_id)) == isbn_id))
def inspect_citekey(citekey):
"""
Check citekeys adhere to expected formats. If an issue is detected a
string describing the issue is returned. Otherwise returns None.
"""
source, identifier = citekey.split(":", 1)
if source == "arxiv":
# https://arxiv.org/help/arxiv_identifier
if not regexes["arxiv"].fullmatch(identifier):
return "arXiv identifiers must conform to syntax described at https://arxiv.org/help/arxiv_identifier."
if source == "pmid":
# https://www.nlm.nih.gov/bsd/mms/medlineelements.html#pmid
if identifier.startswith("PMC"):
return (
"PubMed Identifiers should start with digits rather than PMC. "
f"Should {citekey!r} switch the citation source to 'pmcid'?")
elif not regexes["pmid"].fullmatch(identifier):
return "PubMed Identifiers should be 1-8 digits with no leading zeros."
if source == "pmcid":
# https://www.nlm.nih.gov/bsd/mms/medlineelements.html#pmc
if not identifier.startswith("PMC"):
return "PubMed Central Identifiers must start with 'PMC'."
elif not regexes["pmcid"].fullmatch(identifier):
return ("Identifier does not conform to the PMCID regex. "
"Double check the PMCID.")
if source == "doi":
if identifier.startswith("10."):
# https://www.crossref.org/blog/dois-and-matching-regular-expressions/
if not regexes["doi"].fullmatch(identifier):
return ("Identifier does not conform to the DOI regex. "
"Double check the DOI.")
elif identifier.startswith("10/"):
# shortDOI, see http://shortdoi.org
if not regexes["shortdoi"].fullmatch(identifier):
return ("Identifier does not conform to the shortDOI regex. "
"Double check the shortDOI.")
else:
return "DOIs must start with '10.' (or '10/' for shortDOIs)."
if source == "isbn":
import isbnlib
fail = isbnlib.notisbn(identifier, level="strict")
if fail:
return f"identifier violates the ISBN syntax according to isbnlib v{isbnlib.__version__}"
if source == "wikidata":
# https://www.wikidata.org/wiki/Wikidata:Identifiers
if not identifier.startswith("Q"):
return "Wikidata item IDs must start with 'Q'."
elif not regexes["wikidata"].fullmatch(identifier):
return ("Identifier does not conform to the Wikidata regex. "
"Double check the entity ID.")
return None
def getIsbn():
code = input('Scan book barcode or enter ISBN: ')
if code == 'x': # Exit program when 'x' entered
return False
while isbnlib.notisbn(code):
code = input('\nPlease scan or enter a valid ISBN: ')
if code == 'x': # Exit program when 'x' entered
return False
return code
async def check(self, entry):
isbn = entry.data.get('isbn')
if not isbn:
return []
if notisbn(isbn):
return [("invalid_isbn", "ISBN {} is invalid.".format(isbn), "")]
else:
return []
def clean_isbn(isbn: str) -> str:
"""Cleans ISBN and formats it as ISBN 13
:param isbn: ISBN 10 or ISBN 13
:return: Cleaned ISBN, formatted to ISBN 13 and with hyphens stripped out
"""
if isbnlib.notisbn(isbn):
raise ValueError(f"{isbn} is not a valid ISBN")
return isbnlib.to_isbn13(isbn)
def inspect_citekey(citekey):
"""
Check citekeys adhere to expected formats. If an issue is detected a
string describing the issue is returned. Otherwise returns None.
"""
source, identifier = citekey.split(':', 1)
if source == 'pmid':
# https://www.nlm.nih.gov/bsd/mms/medlineelements.html#pmid
if identifier.startswith('PMC'):
return (
'PubMed Identifiers should start with digits rather than PMC. '
f"Should {citekey!r} switch the citation source to 'pmcid'?")
elif not regexes['pmid'].fullmatch(identifier):
return 'PubMed Identifiers should be 1-8 digits with no leading zeros.'
if source == 'pmcid':
# https://www.nlm.nih.gov/bsd/mms/medlineelements.html#pmc
if not identifier.startswith('PMC'):
return "PubMed Central Identifiers must start with 'PMC'."
elif not regexes['pmcid'].fullmatch(identifier):
return ('Identifier does not conform to the PMCID regex. '
'Double check the PMCID.')
if source == 'doi':
if identifier.startswith('10.'):
# https://www.crossref.org/blog/dois-and-matching-regular-expressions/
if not regexes['doi'].fullmatch(identifier):
return ('Identifier does not conform to the DOI regex. '
'Double check the DOI.')
elif identifier.startswith('10/'):
# shortDOI, see http://shortdoi.org
if not regexes['shortdoi'].fullmatch(identifier):
return ('Identifier does not conform to the shortDOI regex. '
'Double check the shortDOI.')
else:
return ("DOIs must start with '10.' (or '10/' for shortDOIs).")
if source == 'isbn':
import isbnlib
fail = isbnlib.notisbn(identifier, level='strict')
if fail:
return (
f'identifier violates the ISBN syntax according to isbnlib v{isbnlib.__version__}'
)
if source == 'wikidata':
# https://www.wikidata.org/wiki/Wikidata:Identifiers
if not identifier.startswith('Q'):
return ("Wikidata item IDs must start with 'Q'.")
elif not regexes['wikidata'].fullmatch(identifier):
return ('Identifier does not conform to the Wikidata regex. '
'Double check the entity ID.')
return None
def isbn_needs_normalization(isbn: str) -> bool:
"""
Returns True if the given ISBN is valid and needs to be normalized (hyphens removed, letters capitalized, etc.)
Returns False otherwise
"""
if not set(isbn.strip()).issubset(ALLOWED_ISBN_CHARS):
return False
elif isbnlib.notisbn(isbn):
return False
else:
normalized_isbn = isbnlib.get_canonical_isbn(isbn) # get_canonical_isbn returns None if ISBN is invalid
return normalized_isbn and normalized_isbn != isbn
def _is_valid_value(self, data):
value = data['value']
if not value:
return False
key = data['key']
if key == 'isbn':
return not notisbn(value)
if key == 'issn':
return is_valid_issn(value)
if key == 'page-progression':
return value in ['lr', 'rl']
return True
def isbn_validator(isbnlike):
"""
This is a validator for our isbn data. The Book class only accepts isbn13 format, so
if this function receive a isbn10 it will raise a exception.
"""
if (not is_isbn13(isbnlike)) or notisbn(isbnlike):
raise ValidationError("ISBN invalid")
else:
try:
m = meta(canonical(isbnlike))
print(m)
except Exception:
raise ValidationError("ISBN valid but not used")
def get_isbn(self):
_isbn = self.book.get_metadata('DC', 'identifier')
# [('Ratio', {})]
if _isbn:
isbn = _isbn[0][0]
if not isbn:
return None
if isbnlib.notisbn(isbn):
return None
if isbnlib.is_isbn10(isbn):
return isbnlib.to_isbn13(isbn)
return isbn
return None
def isbn_info(request):
isbn = request.GET.get('isbn', '')
data = {}
if isbnlib.notisbn(isbn):
return JsonResponse(data, status=400)
description = isbnlib.desc(isbn)
if description:
data.update({'description': description})
try:
metadata = isbnlib.meta(isbn, 'isbndb')
except Exception:
metadata = {}
data.update({'meta': metadata})
return JsonResponse(data)
def getIsbn():
'''
Request user input of ISBN number
INPUT: None
OUTPUT: Valid ISBN string or False
'''
code = input('Scan book barcode or enter ISBN code: ')
if code == 'x': # Exit program when 'x' entered
return False
while isbnlib.notisbn(code):
code = input('\nPlease scan or enter a valid ISBN code: ')
if code == 'x': # Exit program when 'x' entered
return False
return code
def validate_item(self, item):
key, value = item['key'], item['value']
item['valid'] = True
if key == 'isbn' and notisbn(value) \
or key == 'issn' and not is_valid_issn(value):
if value == "":
pass
else:
item['valid'] = False
item['message'] = 'Must be valid {}'.format(key)
elif key == 'page-progression':
if value not in ['lr', 'rl']:
item['valid'] = False
item['message'] = '{} must be either "rl" or "lr"'.format(key)
def _query_blocking(self, entry, provider):
isbn = entry.data.get('isbn')
# Okay, we're actually going to make a HTTP request
self._ratelimit.get()
if not isbn:
self._ui.finish_subtask('ISBNQuery')
return None
if notisbn(isbn):
self._ui.finish_subtask('ISBNQuery')
return (None, "{} is not a valid ISBN.".format(isbn))
try:
bibtex_data = self._formatter(meta(isbn, service=provider))
except ISBNLibException as e:
self._ui.finish_subtask('ISBNQuery')
return (None, e)
except socket.timeout:
self._ui.finish_subtask('ISBNQuery')
raise RetrievalProblem("Socket timeout during"
" ISBN metadata retrieval")
try:
parsed_data = bibtexparser.loads(bibtex_data)
except:
self._ui.finish_subtask('ISBNQuery')
raise RetrievalProblem("Data from ISBN source could not be parsed")
if len(parsed_data.entries) != 1:
self._ui.finish_subtask('ISBNQuery')
raise RetrievalProblem(
"ISBN search did not return exactly one result.")
retrieved = Entry(parsed_data.entries[0], self._ui)
s = Suggestion("isbn_{}".format(provider), entry)
for (k, v) in retrieved.data.items():
if k.lower() == 'id':
continue
s.add_field(k, v)
for (first, last) in s.authors:
s.add_author(first, last)
for (first, last) in s.editors:
s.add_editor(first, last)
return (s, None)
def preprocess_isbns(isbns):
"""
:param isbns: isbns in different formats
:return: canonical isbn13s
"""
canonical_isbns = []
for isbn in isbns:
if not isbnlib.notisbn(isbn, level='strict'):
if isbnlib.is_isbn10(isbn):
isbn = isbnlib.to_isbn13(isbn)
isbn = isbnlib.get_canonical_isbn(isbn)
canonical_isbns.append(isbn)
canonical_isbns = set(canonical_isbns)
return list(canonical_isbns)
def set_isbn10():
"""if isbn10 is empty, copy from from isbn"""
sql = "select id, isbn from book where isbn_10 = ? and isbn != ?;"
key = ("", "")
cur = conn.cursor()
cur.execute(sql, key)
res = cur.fetchall()
data = []
for book in res:
id, isbn = book
if notisbn(isbn):
print(f"error, id {id}, isbn {isbn} is not a valid isbn")
continue
isbn_10 = isbn
data.append((isbn_10, id))
sql = "UPDATE book SET isbn_10 = ? WHERE id = ?;"
cur.executemany(sql, data)
conn.commit()
def enterISBN(filename):
isbnEntered = False
initWindow = make_init_window()
while True:
initEvent, initValues = initWindow.read()
while not isbnEntered:
if initEvent == sg.WIN_CLOSED or initEvent =='Exit':
isbn = 'Exit'
initWindow.close()
return isbn
if initEvent == 'Display Inventory':
display_CSV(filename)
break
if initEvent == 'Edit Existing Entry':
toEdit = []
isbn = initValues['-ISBN-']
toEdit = find_existing_ISBN(filename, isbn)
if isbn == '':
sg.Popup('Please Enter an ISBN')
break
elif toEdit == []:
sg.Popup('No Existing Entries of this ISBN')
break
else:
make_edit_window(toEdit)
break
if initEvent == 'Continue':
isbn = initValues['-ISBN-']
if isbn == '':
sg.Popup('Please Enter an ISBN')
break
if not isbnlib.notisbn(isbn, level='strict'): # Verify the ISBN is valid
initWindow.close()
return isbn
else:
sg.Popup("Invalid ISBN")
break
def add_book_from_isbn(context, request):
"""
This is a view that will be called when the button for adding books using
their ISBN is clicked from the contents view. It uses ``isbnlib`` to query
some book databases and fetch the metadata if the ISBN exists. Once we have
the metadata, we create a new book and redirect to the contents view.
"""
isbn = request.POST.get('isbn', '')
if isbn:
if isbnlib.notisbn(isbn):
request.session.flash('Not a valid ISBN: %s.' % isbn, 'danger')
elif isbn in context:
request.session.flash('ISBN already exists: %s.' % isbn, 'danger')
else:
metadata = isbnlib.meta(isbn)
if metadata:
year = metadata['Year']
if not year:
year = 0
else:
year = int(year)
book = request.registry.content.create(
'Book',
isbn,
metadata['Title'],
metadata['Authors'],
metadata['Publisher'],
year,
)
context[isbn] = book
request.session.flash('Added book with ISBN: %s.' % isbn,
'success')
else:
request.session.flash('No data exists for ISBN: %s.' % isbn,
'danger')
return HTTPFound(request.sdiapi.mgmt_path(context, '@@contents'))
else:
return {}
def __init__(self, info):
'''
Instantiate Book objects with either ISBN or title
INPUT: String that is either a book title or an ISBN
OUTPUT: None
'''
if isbnlib.notisbn(info): # Instantiate using ISBN
self.title = info
self.isbn = ''
else: # Instantiate using title
self.isbn = info
self.title = ''
# Create all attributes that will be used
self.author = ''
self.publisher = ''
self.year = ''
self.language = ''
self.authorLoc = ''
self.publisherLoc = ''
self.plotLoc = ''
def is_valid(isbn_id):
"""
Check that a given string is a valid ISBN.
:param isbn_id: the isbn to be checked.
:returns: boolean indicating whether the isbn is valid or not.
>>> is_valid("978-3-16-148410-0")
True
>>> is_valid("9783161484100")
True
>>> is_valid("9783161484100aa")
False
>>> is_valid("abcd")
False
>>> is_valid("0136091814")
True
>>> is_valid("0136091812")
False
>>> is_valid("9780136091817")
False
>>> is_valid("123456789X")
True
"""
return (
(not isbnlib.notisbn(isbn_id)) and (
isbnlib.get_canonical_isbn(isbn_id) == isbn_id or
isbnlib.mask(isbnlib.get_canonical_isbn(isbn_id)) == isbn_id)
)
def is_isbn(identifier):
return isbnlib.notisbn(identifier)
# data from: olib, goob, dbkk
data_from = "olib"
# convert isbn with np.nan to empty string
isbn = row["ISBN-nr"]
# isbn = "" if isbn is np.nan else isbn
if isbn is np.nan:
isbn = ""
row["ISBN-nr"] = ""
isbn = canonical(isbn)
# use DBKKs db, just parse the current row
dbkk = dbkkquery(row)
# check if current book exist in db
if notisbn(isbn):
sql_dict = {k: dbkk[k] for k in ("title", "authors")}
else:
sql_dict = {k: dbkk[k] for k in ("isbn", )}
if book_exist(conn, sql_dict):
print("book exist")
continue
if (isbn is not np.nan) and (not notisbn(isbn)):
# lookup the isbn. Query all sources to get most info. Then merge
olib = oquery(isbn)
goob = gquery(isbn)
data_isbn.append(row)
else:
async def convert(self, ctx: commands.Context, argument: str) -> int:
# if argument.isnumeric() and (len(argument)==10 or len(argument)==13):
# return int(argument)
if isbnlib.notisbn(argument):
raise commands.errors.BadArgument('Invalid ISBN: ' + argument)
return isbnlib.get_canonical_isbn(argument)
def match(cls, uri: str) -> Optional[papis.importer.Importer]:
import isbnlib
if isbnlib.notisbn(uri):
return None
return Importer(uri=uri)
def validate_isbn(value):
if notisbn(value):
raise ValidationError('%(value)s is not ISBN-like',
params={'value': value})
