def migrate_isbn(change):
''' two return values: change valid, can be autoapproved '''
issue_rev = change.issuerevisions.get()
cand_isbn = issue_rev.notes[4:].strip(':# ').split()[0].strip('.;')
if isbn.is_valid(cand_isbn):
issue_rev.isbn = cand_isbn
pos = issue_rev.notes.find(cand_isbn) + len(cand_isbn)
new_notes = issue_rev.notes[pos:].lstrip(' ;.\r\n')
if change.storyrevisions.count():
cover_rev = change.storyrevisions.get(sequence_number=0)
if cover_rev.notes == issue_rev.notes:
cover_rev.notes = new_notes
cover_rev.save()
issue_rev.notes = new_notes
issue_rev.save()
change.submit()
if issue_rev.notes == '':
return True, True
else:
return True, False
else:
# no valid isbn, no reason to keep changeset in db
change.discard(anon) # to cleanup issue reserved state
change.delete()
return False, False
def migrate_isbn(change):
''' two return values: change valid, can be autoapproved '''
issue_rev = change.issuerevisions.get()
cand_isbn = issue_rev.notes[4:].strip(':# ').split()[0].strip('.;')
if isbn.is_valid(cand_isbn):
issue_rev.isbn = cand_isbn
pos = issue_rev.notes.find(cand_isbn) + len(cand_isbn)
new_notes = issue_rev.notes[pos:].lstrip(' ;.\r\n')
if change.storyrevisions.count():
cover_rev = change.storyrevisions.get(sequence_number=0)
if cover_rev.notes == issue_rev.notes:
cover_rev.notes = new_notes
cover_rev.save()
issue_rev.notes = new_notes
issue_rev.save()
change.submit()
if issue_rev.notes == '':
return True, True
else:
return True, False
else:
# no valid isbn, no reason to keep changeset in db
change.discard(anon) # to cleanup issue reserved state
change.delete()
return False, False
def normalize_isbn_map(data):
normalized = {}
for k, v in data.items():
if isbn_tool.is_valid(k):
compact_isbn = isbn_tool.compact(k)
normalized[compact_isbn] = v
return normalized
def create_isbn(self, key, var_list=None):
tmp_list = []
for var in var_list:
# print(self.csvRow[var].strip())
tmp_elem = re.split('; |, ', self.csvRow[var].strip())
for i in range(len(tmp_elem)):
"""
Filter ISSN and ISBN only
"""
tmp_var_isbn = self.remove_non_isbn_chars(tmp_elem[i].upper())
if isbn.is_valid(tmp_var_isbn):
# print("Valid ISBN: " + tmp_var_isbn)
tmp_elem[i] = isbn.format(tmp_var_isbn)
else:
tmp_elem[i] = None
# Filter all elements, remove text chars, parenthesis etc
# print(tmp_elem)
tmp_list += tmp_elem
# print(tmp_list)
tmp_list = list(filter(None, tmp_list)) # remove empty list elements
# print(tmp_list)
# No semi-colon, so do a join here. Cannot use the function to split semi-colons
return '||'.join(filter(None, tmp_list)).strip()
def migrate_isbn_series_issue(change_series, change_issue):
''' two return values: change valid, can be autoapproved '''
series_rev = change_series.seriesrevisions.get()
issue_rev = change_issue.issuerevisions.get()
if series_rev.notes.startswith('ISBN-10'):
if series_rev.notes.find('ISBN-13') < 0:
cand_isbn = series_rev.notes[7:].strip(':# ').split()[0].strip(
'.;')
else:
return False, False, False
elif series_rev.notes.startswith('ISBN-13'):
if series_rev.notes.find('ISBN-10') < 0:
cand_isbn = series_rev.notes[7:].strip(':# ').split()[0].strip(
'.;')
else:
return False, False, False
else:
cand_isbn = series_rev.notes[4:].strip(':# ').split()[0].strip('.;')
if isbn.is_valid(cand_isbn):
issue_rev.isbn = cand_isbn
pos = series_rev.notes.find(cand_isbn) + len(cand_isbn)
new_notes = series_rev.notes[pos:].lstrip(' ;.\r\n')
series_rev.notes = new_notes
series_rev.save()
issue_rev.save()
change_series.submit()
change_issue.submit()
if series_rev.notes == '':
return True, True, True
else:
return True, False, True
else:
return False, False, False
def validate_isbn(item, errors):
"""
Checks ISBN for validity.
Will accept both ISBN-10 and ISBN-13 formats
"""
isbn_list = item.get("isbn")
if isbn_list is None:
return
for idx, single_isbn in enumerate(isbn_list):
if not isbn.is_valid(single_isbn):
errors.add("ISBN #{idx} isn't valid".format(
idx=idx
))
continue
formatted = isbn.format(single_isbn)
if (formatted != single_isbn):
errors.add("ISBN #{idx} ({single_isbn}) should be reformatted to {formatted}".format(
idx=idx,
single_isbn=single_isbn,
formatted=formatted
))
if (isbn.isbn_type(single_isbn) != 'ISBN13'):
errors.add("ISBN-10 #{idx} ({single_isbn}) should be reformatted to ISBN-13 {formatted}".format(
idx=idx,
single_isbn=single_isbn,
formatted=isbn.to_isbn13(single_isbn)
))
def migrate_isbn_series_issue(change_series, change_issue):
''' two return values: change valid, can be autoapproved '''
series_rev = change_series.seriesrevisions.get()
issue_rev = change_issue.issuerevisions.get()
if series_rev.notes.startswith('ISBN-10'):
if series_rev.notes.find('ISBN-13') < 0:
cand_isbn = series_rev.notes[7:].strip(':# ').split()[0].strip('.;')
else:
return False, False, False
elif series_rev.notes.startswith('ISBN-13'):
if series_rev.notes.find('ISBN-10') < 0:
cand_isbn = series_rev.notes[7:].strip(':# ').split()[0].strip('.;')
else:
return False, False, False
else:
cand_isbn = series_rev.notes[4:].strip(':# ').split()[0].strip('.;')
if isbn.is_valid(cand_isbn):
issue_rev.isbn = cand_isbn
pos = series_rev.notes.find(cand_isbn) + len(cand_isbn)
new_notes = series_rev.notes[pos:].lstrip(' ;.\r\n')
series_rev.notes = new_notes
series_rev.save()
issue_rev.save()
change_series.submit()
change_issue.submit()
if series_rev.notes == '':
return True, True, True
else:
return True, False, True
else:
return False, False, False
def check_isbn(item, errors):
"""
Checks ISBN for validity.
Will accept both ISBN-10 and ISBN-13 formats
"""
isbn_list = item.get("isbn")
if isbn_list is None:
return
for idx, single_isbn in enumerate(isbn_list):
if not isbn.is_valid(single_isbn):
errors.add("ISBN #{idx} isn't valid".format(
idx=idx
))
continue
formatted = isbn.format(single_isbn)
if (formatted != single_isbn):
errors.add("ISBN #{idx} ({single_isbn}) should be reformatted to {formatted}".format(
idx=idx,
single_isbn=single_isbn,
formatted=formatted
))
if (isbn.isbn_type(single_isbn) != 'ISBN13'):
errors.add("ISBN-10 #{idx} ({single_isbn}) should be reformatted to ISBN-13 {formatted}".format(
idx=idx,
single_isbn=single_isbn,
formatted=isbn.to_isbn13(single_isbn)
))
def show_isbn(_isbn):
isbn = ''
for code in _isbn.split(';'):
if stdisbn.is_valid(code):
isbn += stdisbn.format(code) + '; '
else:
isbn += code + '; '
if isbn:
# chop trailing '; '
return isbn[:-2]
def show_isbn(_isbn):
isbn = ''
for code in _isbn.split(';'):
if stdisbn.is_valid(code):
isbn += stdisbn.format(code) + '; '
else:
isbn += code + '; '
if isbn:
# chop trailing '; '
return isbn[:-2]
def ISBNValidator(value):
""" Check string is a valid ISBN number"""
if not isinstance(value, string_types):
raise ValidationError(_(u'Invalid ISBN: Not a string'))
if len(value) != 10 and len(value) != 13:
raise ValidationError(_(u'Invalid ISBN: Wrong length'))
if not isbn.is_valid(value):
raise ValidationError(_(u'Invalid ISBN: Failed checksum'))
def ISBNValidator(value):
""" Check string is a valid ISBN number"""
if not isinstance(value, string_types):
raise ValidationError(_(u'No es un numero ISBN valido'))
if len(value) != 10 and len(value) != 13:
raise ValidationError(_(u'Número ISBN inválido: Longitud incorrecta.'))
if not isbn.is_valid(value):
raise ValidationError(_(u'Número ISBN inválido: No se pudo validar.'))
def ISBNValidator(value):
""" Check string is a valid ISBN number"""
if not isinstance(value, string_types):
raise ValidationError(_(u'Invalid ISBN: Not a string'))
if len(value) != 10 and len(value) != 13:
raise ValidationError(_(u'Invalid ISBN: Wrong length'))
if not isbn.is_valid(value):
raise ValidationError(_(u'Invalid ISBN: Failed checksum'))
def ISBNValidator(value):
""" Check string is a valid ISBN number"""
if not isinstance(value, string_types):
raise ValidationError(_(u'No es un numero ISBN valido'))
if len(value) != 10 and len(value) != 13:
raise ValidationError(_(u'Número ISBN inválido: Longitud incorrecta.'))
if not isbn.is_valid(value):
raise ValidationError(_(u'Número ISBN inválido: No se pudo validar.'))
def ISBNValidator(raw_isbn):
isbn_to_check = raw_isbn.replace('-', '').replace(' ', '')
if len(isbn_to_check) != 10 and len(isbn_to_check) != 13:
raise ValidationError(_(u'Invalid ISBN: Wrong length'))
if not isbn.is_valid(isbn_to_check):
raise ValidationError(_(u'Invalid ISBN: Failed checksum'))
if isbn_to_check != isbn_to_check.upper():
raise ValidationError(_(u'Invalid ISBN: Only upper case allowed'))
return True
def compute_isbn_qobj(isbn, prefix, op):
if stdisbn.is_valid(isbn):
isbn_compact = stdisbn.compact(isbn)
q_obj = Q(**{ '%svalid_isbn' % prefix: isbn_compact})
# need to search for both ISBNs to be safe
if stdisbn.isbn_type(isbn_compact) == 'ISBN13' and \
isbn_compact.startswith('978'):
isbn10 = isbn_compact[3:-1]
isbn10 += stdisbn._calc_isbn10_check_digit(isbn10)
q_obj |= Q(**{ '%svalid_isbn' % prefix: isbn10})
elif stdisbn.isbn_type(isbn_compact) == 'ISBN10':
q_obj |= Q(**{ '%svalid_isbn' % prefix:
stdisbn.to_isbn13(isbn_compact)})
else:
q_obj = Q(**{ '%sisbn__%s' % (prefix, op): isbn})
return q_obj
def compute_isbn_qobj(isbn, prefix, op):
if stdisbn.is_valid(isbn):
isbn_compact = stdisbn.compact(isbn)
q_obj = Q(**{'%svalid_isbn' % prefix: isbn_compact})
# need to search for both ISBNs to be safe
if stdisbn.isbn_type(isbn_compact) == 'ISBN13' and \
isbn_compact.startswith('978'):
isbn10 = isbn_compact[3:-1]
isbn10 += stdisbn._calc_isbn10_check_digit(isbn10)
q_obj |= Q(**{'%svalid_isbn' % prefix: isbn10})
elif stdisbn.isbn_type(isbn_compact) == 'ISBN10':
q_obj |= Q(
**{'%svalid_isbn' % prefix: stdisbn.to_isbn13(isbn_compact)})
else:
q_obj = Q(**{'%sisbn__%s' % (prefix, op): isbn})
return q_obj
def ISBNValidator(raw_isbn):
""" Check string is a valid ISBN number"""
isbn_to_check = raw_isbn.replace('-', '').replace(' ', '')
if not isinstance(isbn_to_check, string_types):
raise ValidationError(_(u'Invalid ISBN: Not a string'))
if len(isbn_to_check) != 10 and len(isbn_to_check) != 13:
raise ValidationError(_(u'Invalid ISBN: Wrong length'))
if not isbn.is_valid(isbn_to_check):
raise ValidationError(_(u'Invalid ISBN: Failed checksum'))
if isbn_to_check != isbn_to_check.upper():
raise ValidationError(_(u'Invalid ISBN: Only upper case allowed'))
return True
def ISBNValidator(raw_isbn):
""" Check string is a valid ISBN number"""
isbn_to_check = raw_isbn.replace('-', '').replace(' ', '')
if not isinstance(isbn_to_check, string_types):
raise ValidationError(_(u'Invalid ISBN: Not a string'))
if len(isbn_to_check) != 10 and len(isbn_to_check) != 13:
raise ValidationError(_(u'Invalid ISBN: Wrong length'))
if not isbn.is_valid(isbn_to_check):
raise ValidationError(_(u'Invalid ISBN: Failed checksum'))
if isbn_to_check != isbn_to_check.upper():
raise ValidationError(_(u'Invalid ISBN: Only upper case allowed'))
return True
def formatIdentifier(stringToFormat, type):
"""
This is used to format/check ISBN and ISSN numbers
"""
if type.lower() == "isbn":
if isbn.is_valid(stringToFormat):
log.debug("OK ISBN: {0}".format(stringToFormat))
return isbn.format(stringToFormat)
else:
log.warning("Malformed ISBN: {0}".format(stringToFormat))
return stringToFormat
elif type.lower() == "issn":
if issn.is_valid(stringToFormat):
log.debug("OK ISSN: {0}".format(stringToFormat))
return issn.format(stringToFormat)
else:
log.warning("Malformed ISSN: {0}".format(stringToFormat))
return stringToFormat
def isbn(field):
"""Check if an ISBN is valid.
Prints the ISBN if invalid.
stdnum's is_valid() function never raises an exception.
See: https://arthurdejong.org/python-stdnum/doc/1.11/index.html#stdnum.module.is_valid
"""
# Skip fields with missing values
if pd.isna(field):
return
# Try to split multi-value field on "||" separator
for value in field.split("||"):
if not stdnum_isbn.is_valid(value):
print(f"{Fore.RED}Invalid ISBN: {Fore.RESET}{value}")
return
def check_isbn(raw_isbn):
"""If the ISBN can be worked on, return True"""
if len(raw_isbn) == 17 or not isbn.is_valid(raw_isbn):
return False
else:
return True
books_isbn13s = books["isbn13"].unique()
covers_isbn13s = covers["isbn"].unique()
isbns = set(bx_books_isbns.tolist() + bx_ratings_isbns.tolist() + books_isbns.tolist())
isbn13s = set(books_isbn13s.tolist() + covers_isbn13s.tolist())
isbn_strings = list()
for i in covers_isbn13s.tolist():
if isinstance(i, int):
isbn_strings.append(str(i))
isbn13s = set(books_isbn13s.tolist() + isbn_strings)
invalid = set()
for num in isbns.union(isbn13s):
if not isbn.is_valid(num):
invalid.add(num)
bx_ratings_isbns = set(bx_ratings_isbns)
bx_books_isbns = set(bx_books_isbns)
covers["isbn"] = covers["isbn"].astype(str)
for num in covers["isbn"]:
if (not isbn.is_valid(num) and num not in invalid):
print(num)
bx_books_idx = bookx_books.index[bookx_books["ISBN"].isin(set(bx_books_isbns) - invalid)]
isbn_to_apply_books = bookx_books.loc[bx_books_idx, "ISBN"]
bx_ratings_idx = bookx_ratings.index[bookx_ratings["ISBN"].isin(set(bx_ratings_isbns) - invalid)]
def handle_file(file, root, relaxng):
"""Handle one ABACUS file. Return 1 if the import was successful, 0 otherwise."""
# Check the validity of the given XML file
tree = etree.parse(file)
relaxng.assertValid(tree)
evaluator = etree.XPathEvaluator(tree)
get_key = make_get_key_fn(evaluator)
# fetch the data from the XML file
product_number = get_key("%s/artikel_nr" % root)
params = fetch_params(get_key, root)
logger.info('Processing "%s" [%s]...', params['title'], product_number)
# validate the ISBN but allow for some weird exceptions (same as in schema)
if not (params['source'] == '' or
params['source'] == 'keine' or
re.match('SBS[0-9]{6}', params['source']) or
is_valid(params['source'])):
raise ValidationError(params['source'])
# If the XML indicates that this product is not produced with Daisy Producer ignore this file
daisy_producer = get_key("%s/MetaData/sbs/daisy_producer" % root)
if daisy_producer != "ja":
logger.debug('Ignoring "%s" as daisy_producer is set to "%s"',
params['title'], daisy_producer)
return 0
# validate the product number
product_type = get_type(product_number)
if product_type == None:
raise ValidationError(product_number)
product_number_has_been_seen_before = False
if get_documents_by_product_number(product_number):
# If the order has been imported before just update the meta data of the existing order
product_number_has_been_seen_before = True
documents = get_documents_by_product_number(product_number)
# there should only ever be one document here. Make sure this is so
if len(documents) > 1:
logger.error('There is more than one document for the given product %s (%s)',
product_number, documents)
document = documents[0]
logger.debug('Document "%s" for order number "%s" has already been imported.',
document.title, product_number)
document = update_document(document, params)
elif get_documents_by_source_or_title_source_edition(
params['source'], params['title'], params['source_edition']):
# check if the book has been produced for another order
documents = get_documents_by_source_or_title_source_edition(
params['source'], params['title'], params['source_edition'])
# what if there are multiple documents that match the query?
if len(documents) > 1:
logger.error('There is more than one document for the given source [%s] or title and source_edition [%s, %s] (%s)',
params['source'], params['title'],
params['source_edition'], documents)
document = documents[0]
logger.debug('Document "%s" has already been imported for a different product.',
document.title)
document = update_document(document, params)
# update the product association
logger.debug('Updating product association ["%s" -> "%s"].', document.title, product_number)
Product.objects.create(identifier=product_number, type=product_type, document=document)
else:
# If the the order hasn't been imported before create the new order
logger.debug('Document "%s" has not yet been imported. Creating document for product "%s".',
params['title'], product_number)
# create and save the document
if product_type == 3:
# etext books should have 'in_production' as initial state
try:
params['state'] = State.objects.get(name='in_production')
except State.DoesNotExist:
params['state'] = State.objects.order_by('sort_order')[0]
document = Document.objects.create(**params)
# create the product association
Product.objects.create(identifier=product_number, type=product_type, document=document)
# create an empty xml
update_xml_with_metadata(document)
# if the product_number has never been seen before then we are talking about a new
# production, i.e. try to check out the document in the archive
# if not product_number_has_been_seen_before:
# checkout_document(product_number)
# If the order has been archived before fetch the xml from the archive
fetch_xml(document, product_number, not product_number_has_been_seen_before)
return 1
def check_code_isbn(number):
'''
Check isbn code.
'''
return isbn.is_valid(number)
