def test_sorting(self):
" Test the various sorting APIs "
german = """Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse""".split()
german_good = (
"""Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag""".split()
)
french = """dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché""".split()
french_good = """bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché""".split() # noqa
# Test corner cases
sort_key = icu.sort_key
s = "\U0001f431"
self.ae(
sort_key(s),
sort_key(s.encode(sys.getdefaultencoding())),
"UTF-8 encoded object not correctly decoded to generate sort key",
)
self.ae(s.encode("utf-16"), s.encode("utf-16"), "Undecodable bytestring not returned as itself")
self.ae(b"", sort_key(None))
self.ae(0, icu.strcmp(None, b""))
self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding())))
# Test locales
with make_collation_func("dsk", "de", func="sort_key") as dsk:
self.ae(german_good, sorted(german, key=dsk))
with make_collation_func("dcmp", "de", template="_strcmp_template") as dcmp:
for x in german:
for y in german:
self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y))
with make_collation_func("fsk", "fr", func="sort_key") as fsk:
self.ae(french_good, sorted(french, key=fsk))
with make_collation_func("fcmp", "fr", template="_strcmp_template") as fcmp:
for x in french:
for y in french:
self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y))
with make_collation_func("ssk", "es", func="sort_key") as ssk:
self.assertNotEqual(ssk("peña"), ssk("pena"))
with make_collation_func("scmp", "es", template="_strcmp_template") as scmp:
self.assertNotEqual(0, scmp("pena", "peña"))
for k, v in {"pèché": "peche", "flüße": "Flusse", "Štepánek": "ŠtepaneK"}.iteritems():
self.ae(0, icu.primary_strcmp(k, v))
# Test different types of collation
self.ae(icu.primary_sort_key("Aä"), icu.primary_sort_key("aa"))
self.assertLess(icu.numeric_sort_key("something 2"), icu.numeric_sort_key("something 11"))
self.assertLess(icu.case_sensitive_sort_key("A"), icu.case_sensitive_sort_key("a"))
self.ae(0, icu.strcmp("a", "A"))
self.ae(cmp("a", "A"), icu.case_sensitive_strcmp("a", "A"))
self.ae(0, icu.primary_strcmp("ä", "A"))
def add_category(self):
self.save_category()
cat_name = unicode_type(self.input_box.text()).strip()
if cat_name == '':
return False
comps = [c.strip() for c in cat_name.split('.') if c.strip()]
if len(comps) == 0 or '.'.join(comps) != cat_name:
error_dialog(self, _('Invalid name'),
_('That name contains leading or trailing periods, '
'multiple periods in a row or spaces before '
'or after periods.')).exec_()
return False
for c in sorted(self.categories.keys(), key=sort_key):
if strcmp(c, cat_name) == 0 or \
(icu_lower(cat_name).startswith(icu_lower(c) + '.') and
not cat_name.startswith(c + '.')):
error_dialog(self, _('Name already used'),
_('That name is already used, perhaps with different case.')).exec_()
return False
if cat_name not in self.categories:
self.category_box.clear()
self.current_cat_name = cat_name
self.categories[cat_name] = []
self.applied_items = []
self.populate_category_list()
self.input_box.clear()
self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
return True
def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
v = strcmp(x, y)
if v < 0:
return lt
if v == 0:
return eq
return gt
def add_category(self):
self.save_category()
cat_name = str(self.input_box.text()).strip()
if cat_name == '':
return False
comps = [c.strip() for c in cat_name.split('.') if c.strip()]
if len(comps) == 0 or '.'.join(comps) != cat_name:
error_dialog(
self, _('Invalid name'),
_('That name contains leading or trailing periods, '
'multiple periods in a row or spaces before '
'or after periods.')).exec_()
return False
for c in sorted(list(self.categories.keys()), key=sort_key):
if strcmp(c, cat_name) == 0 or \
(icu_lower(cat_name).startswith(icu_lower(c) + '.') and
not cat_name.startswith(c + '.')):
error_dialog(
self, _('Name already used'),
_('That name is already used, perhaps with different case.'
)).exec_()
return False
if cat_name not in self.categories:
self.category_box.clear()
self.current_cat_name = cat_name
self.categories[cat_name] = []
self.applied_items = []
self.populate_category_list()
self.input_box.clear()
self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
return True
def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):
v = strcmp(x, y)
if v < 0:
return lt
if v == 0:
return eq
return gt
def rename_category(self):
self.save_category()
cat_name = str(self.input_box.text()).strip()
if cat_name == '':
return False
if not self.current_cat_name:
return False
comps = [c.strip() for c in cat_name.split('.') if c.strip()]
if len(comps) == 0 or '.'.join(comps) != cat_name:
error_dialog(
self, _('Invalid name'),
_('That name contains leading or trailing periods, '
'multiple periods in a row or spaces before '
'or after periods.')).exec_()
return False
for c in self.categories:
if strcmp(c, cat_name) == 0:
error_dialog(
self, _('Name already used'),
_('That name is already used, perhaps with different case.'
)).exec_()
return False
# The order below is important because of signals
self.categories[cat_name] = self.categories[self.current_cat_name]
del self.categories[self.current_cat_name]
self.current_cat_name = None
self.populate_category_list()
self.input_box.clear()
self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
return True
def rename_category(self):
self.save_category()
cat_name = unicode_type(self.input_box.text()).strip()
if cat_name == '':
return False
if not self.current_cat_name:
return False
comps = [c.strip() for c in cat_name.split('.') if c.strip()]
if len(comps) == 0 or '.'.join(comps) != cat_name:
error_dialog(self, _('Invalid name'),
_('That name contains leading or trailing periods, '
'multiple periods in a row or spaces before '
'or after periods.')).exec_()
return False
for c in self.categories:
if strcmp(c, cat_name) == 0:
error_dialog(self, _('Name already used'),
_('That name is already used, perhaps with different case.')).exec_()
return False
# The order below is important because of signals
self.categories[cat_name] = self.categories[self.current_cat_name]
del self.categories[self.current_cat_name]
self.current_cat_name = None
self.populate_category_list()
self.input_box.clear()
self.category_box.setCurrentIndex(self.category_box.findText(cat_name))
return True
def test_sorting(self):
' Test the various sorting APIs '
german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split()
german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split()
french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split()
french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split() # noqa
# Test corner cases
sort_key = icu.sort_key
s = '\U0001f431'
self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key')
self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself')
self.ae(b'', sort_key(None))
self.ae(0, icu.strcmp(None, b''))
self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding())))
# Test locales
with make_collation_func('dsk', 'de', func='sort_key') as dsk:
self.ae(german_good, sorted(german, key=dsk))
with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp:
for x in german:
for y in german:
self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y))
with make_collation_func('fsk', 'fr', func='sort_key') as fsk:
self.ae(french_good, sorted(french, key=fsk))
with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp:
for x in french:
for y in french:
self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y))
with make_collation_func('ssk', 'es', func='sort_key') as ssk:
self.assertNotEqual(ssk('peña'), ssk('pena'))
with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp:
self.assertNotEqual(0, scmp('pena', 'peña'))
for k, v in iteritems({u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}):
self.ae(0, icu.primary_strcmp(k, v))
# Test different types of collation
self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa'))
self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11'))
self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a'))
self.ae(0, icu.strcmp('a', 'A'))
self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A'))
self.ae(0, icu.primary_strcmp('ä', 'A'))
def test_sorting(self):
' Test the various sorting APIs '
german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split()
german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split()
french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split()
french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split() # noqa
# Test corner cases
sort_key = icu.sort_key
s = '\U0001f431'
self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key')
self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself')
self.ae(b'', sort_key(None))
self.ae(0, icu.strcmp(None, b''))
self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding())))
# Test locales
with make_collation_func('dsk', 'de', func='sort_key') as dsk:
self.ae(german_good, sorted(german, key=dsk))
with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp:
for x in german:
for y in german:
self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y))
with make_collation_func('fsk', 'fr', func='sort_key') as fsk:
self.ae(french_good, sorted(french, key=fsk))
with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp:
for x in french:
for y in french:
self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y))
with make_collation_func('ssk', 'es', func='sort_key') as ssk:
self.assertNotEqual(ssk('peña'), ssk('pena'))
with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp:
self.assertNotEqual(0, scmp('pena', 'peña'))
for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems():
self.ae(0, icu.primary_strcmp(k, v))
# Test different types of collation
self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa'))
self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11'))
self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a'))
self.ae(0, icu.strcmp('a', 'A'))
self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A'))
self.ae(0, icu.primary_strcmp('ä', 'A'))
def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):
l = [v.strip() for v in val.split(sep) if v.strip()]
c = [v.strip() for v in str.split(sep) if v.strip()]
if l:
for v in l:
for t in c:
if strcmp(t, v) == 0:
return fv
return nfv
def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv):
l = [v.strip() for v in val.split(sep) if v.strip()]
c = [v.strip() for v in str.split(sep) if v.strip()]
if l:
for v in l:
for t in c:
if strcmp(t, v) == 0:
return fv
return nfv
def update_state(self, *args):
au = unicode(self.authors_edit.text())
au = re.sub(r'\s+et al\.$', '', au)
au = self.db.author_sort_from_authors(string_to_authors(au))
normal = strcmp(au, self.current_val) == 0
if normal:
col = 'rgb(0, 255, 0, 20%)'
else:
col = 'rgb(255, 0, 0, 20%)'
self.setStyleSheet('QLineEdit { color: black; '
'background-color: %s; }' % col)
tt = self.tooltips[0 if normal else 1]
self.setToolTip(tt)
self.setWhatsThis(tt)
def update_state(self, *args):
au = unicode(self.authors_edit.text())
au = re.sub(r'\s+et al\.$', '', au)
au = self.db.author_sort_from_authors(string_to_authors(au))
normal = strcmp(au, self.current_val) == 0
if normal:
col = 'rgb(0, 255, 0, 20%)'
else:
col = 'rgb(255, 0, 0, 20%)'
self.setStyleSheet('QLineEdit { color: black; '
'background-color: %s; }'%col)
tt = self.tooltips[0 if normal else 1]
self.setToolTip(tt)
self.setWhatsThis(tt)
def none_cmp(xx, yy):
x = xx[1]
y = yy[1]
if x is None and y is None:
# No sort_key needed here, because defaults are ascii
return cmp(xx[2], yy[2])
if x is None:
return 1
if y is None:
return -1
if isinstance(x, basestring) and isinstance(y, basestring):
c = strcmp(force_unicode(x), force_unicode(y))
else:
c = cmp(x, y)
if c != 0:
return c
# same as above -- no sort_key needed here
return cmp(xx[2], yy[2])
def none_cmp(xx, yy):
x = xx[1]
y = yy[1]
if x is None and y is None:
# No sort_key needed here, because defaults are ascii
return cmp(xx[2], yy[2])
if x is None:
return 1
if y is None:
return -1
if isinstance(x, basestring) and isinstance(y, basestring):
c = strcmp(force_unicode(x), force_unicode(y))
else:
c = cmp(x, y)
if c != 0:
return c
# same as above -- no sort_key needed here
return cmp(xx[2], yy[2])
def many_many(book_id_val_map, db, field, allow_case_change, *args):
dirtied = set()
m = field.metadata
table = field.table
dt = m['datatype']
is_authors = field.name == 'authors'
# Map values to db ids, including any new values
kmap = safe_lower if dt == 'text' else lambda x:x
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
if len(rid_map) != len(table.id_map):
# table has some entries that differ only in case, fix it
table.fix_case_duplicates(db)
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
val_map = {}
case_changes = {}
book_id_val_map = {k:uniq(vals, kmap) for k, vals in book_id_val_map.iteritems()}
for vals in book_id_val_map.itervalues():
for val in vals:
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
case_changes, val_map, is_authors=is_authors)
if case_changes:
change_case(case_changes, dirtied, db, table, m, is_authors=is_authors)
if is_authors:
for item_id, val in case_changes.iteritems():
for book_id in table.col_book_map[item_id]:
current_sort = field.db_author_sort_for_book(book_id)
new_sort = field.author_sort_for_book(book_id)
if strcmp(current_sort, new_sort) == 0:
# The sort strings differ only by case, update the db
# sort
field.author_sort_field.writer.set_books({book_id:new_sort}, db)
book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
for k, vals in book_id_val_map.iteritems()}
# Ignore those items whose value is the same as the current value
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
if v != table.book_col_map.get(k, None)}
dirtied |= set(book_id_item_id_map)
# Update the book->col and col->book maps
deleted = set()
updated = {}
for book_id, item_ids in book_id_item_id_map.iteritems():
old_item_ids = table.book_col_map.get(book_id, None)
if old_item_ids:
for old_item_id in old_item_ids:
table.col_book_map[old_item_id].discard(book_id)
if item_ids:
table.book_col_map[book_id] = item_ids
for item_id in item_ids:
table.col_book_map[item_id].add(book_id)
updated[book_id] = item_ids
else:
table.book_col_map.pop(book_id, None)
deleted.add(book_id)
# Update the db link table
if deleted:
db.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
((k,) for k in deleted))
if updated:
vals = (
(book_id, val) for book_id, vals in updated.iteritems()
for val in vals
)
db.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
((k,) for k in updated))
db.executemany('INSERT INTO {0}(book,{1}) VALUES(?, ?)'.format(
table.link_table, m['link_column']), vals)
if is_authors:
aus_map = {book_id:field.author_sort_for_book(book_id) for book_id
in updated}
field.author_sort_field.writer.set_books(aus_map, db)
# Remove no longer used items
remove = {item_id for item_id in table.id_map if not
table.col_book_map.get(item_id, False)}
if remove:
db.executemany('DELETE FROM %s WHERE id=?'%m['table'],
((item_id,) for item_id in remove))
for item_id in remove:
del table.id_map[item_id]
table.col_book_map.pop(item_id, None)
if is_authors:
table.asort_map.pop(item_id, None)
table.alink_map.pop(item_id, None)
return dirtied
def update_state_and_val(self):
# Handle case change if the authors box changed
aus = authors_to_sort_string(self.authors_edit.current_val)
if strcmp(aus, self.current_val) == 0:
self.current_val = aus
self.update_state()
class _Interpreter(object):
def error(self, message):
m = 'Interpreter: ' + message
raise ValueError(m)
def program(self, funcs, parent, prog, val):
self.parent = parent
self.parent_kwargs = parent.kwargs
self.parent_book = parent.book
self.funcs = funcs
self.locals = {'$': val}
return self.expression_list(prog)
def expression_list(self, prog):
val = ''
for p in prog:
val = self.expr(p)
return val
INFIX_STRING_OPS = {
"==": lambda x, y: strcmp(x, y) == 0,
"!=": lambda x, y: strcmp(x, y) != 0,
"<": lambda x, y: strcmp(x, y) < 0,
"<=": lambda x, y: strcmp(x, y) <= 0,
">": lambda x, y: strcmp(x, y) > 0,
">=": lambda x, y: strcmp(x, y) >= 0,
}
def do_node_string_infix(self, prog):
try:
left = self.expr(prog.left)
right = self.expr(prog.right)
return ('1' if self.INFIX_STRING_OPS[prog.operator](left, right)
else '')
except:
self.error(
_('Error during string comparison. Operator {0}').format(
prog.operator))
INFIX_NUMERIC_OPS = {
"==#": lambda x, y: x == y,
"!=#": lambda x, y: x != y,
"<#": lambda x, y: x < y,
"<=#": lambda x, y: x <= y,
">#": lambda x, y: x > y,
">=#": lambda x, y: x >= y,
}
def float_deal_with_none(self, v):
# Undefined values and the string 'None' are assumed to be zero.
# The reason for string 'None': raw_field returns it for undefined values
return float(v if v and v != 'None' else 0)
def do_node_numeric_infix(self, prog):
try:
left = self.float_deal_with_none(self.expr(prog.left))
right = self.float_deal_with_none(self.expr(prog.right))
return '1' if self.INFIX_NUMERIC_OPS[prog.operator](left,
right) else ''
except:
self.error(
_('Value used in comparison is not a number. Operator {0}').
format(prog.operator))
def do_node_if(self, prog):
test_part = self.expr(prog.condition)
if test_part:
return self.expression_list(prog.then_part)
elif prog.else_part:
return self.expression_list(prog.else_part)
return ''
def do_node_rvalue(self, prog):
try:
return self.locals[prog.name]
except:
self.error(_('Unknown identifier {0}').format(prog.name))
def do_node_func(self, prog):
args = list()
for arg in prog.expression_list:
# evaluate the expression (recursive call)
args.append(self.expr(arg))
# Evaluate the function.
id_ = prog.name.strip()
cls = self.funcs[id_]
return cls.eval_(self.parent, self.parent_kwargs, self.parent_book,
self.locals, *args)
def do_node_constant(self, prog):
return prog.value
def do_node_field(self, prog):
try:
name = self.expr(prog.expression)
try:
return self.parent.get_value(name, [], self.parent_kwargs)
except:
self.error(_('Unknown field {0}').format(name))
except ValueError as e:
raise e
except:
self.error(_('Unknown field {0}').format('parse error'))
def do_node_raw_field(self, prog):
try:
name = self.expr(prog.expression)
res = getattr(self.parent_book, name, None)
if res is not None:
if isinstance(res, list):
fm = self.parent_book.metadata_for_field(name)
if fm is None:
return ', '.join(res)
return fm['is_multiple']['list_to_ui'].join(res)
return unicode_type(res)
except ValueError as e:
raise e
except:
self.error(_('Unknown field {0}').format('parse error'))
def do_node_assign(self, prog):
t = self.expr(prog.right)
self.locals[prog.left] = t
return t
NODE_OPS = {
Node.NODE_IF: do_node_if,
Node.NODE_ASSIGN: do_node_assign,
Node.NODE_CONSTANT: do_node_constant,
Node.NODE_RVALUE: do_node_rvalue,
Node.NODE_FUNC: do_node_func,
Node.NODE_FIELD: do_node_field,
Node.NODE_RAW_FIELD: do_node_raw_field,
Node.NODE_STRING_INFIX: do_node_string_infix,
Node.NODE_NUMERIC_INFIX: do_node_numeric_infix,
}
def expr(self, prog):
try:
return self.NODE_OPS[prog.node_type](self, prog)
except ValueError as e:
raise e
except:
if (DEBUG):
traceback.print_exc()
self.error(_('Internal error evaluating an expression'))
def many_many(book_id_val_map, db, field, allow_case_change, *args):
dirtied = set()
m = field.metadata
table = field.table
dt = m['datatype']
is_authors = field.name == 'authors'
# Map values to db ids, including any new values
kmap = safe_lower if dt == 'text' else lambda x:x
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
if len(rid_map) != len(table.id_map):
# table has some entries that differ only in case, fix it
table.fix_case_duplicates(db)
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
val_map = {}
case_changes = {}
book_id_val_map = {k:uniq(vals, kmap) for k, vals in book_id_val_map.iteritems()}
for vals in book_id_val_map.itervalues():
for val in vals:
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
case_changes, val_map, is_authors=is_authors)
if case_changes:
change_case(case_changes, dirtied, db, table, m, is_authors=is_authors)
if is_authors:
for item_id, val in case_changes.iteritems():
for book_id in table.col_book_map[item_id]:
current_sort = field.db_author_sort_for_book(book_id)
new_sort = field.author_sort_for_book(book_id)
if strcmp(current_sort, new_sort) == 0:
# The sort strings differ only by case, update the db
# sort
field.author_sort_field.writer.set_books({book_id:new_sort}, db)
book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
for k, vals in book_id_val_map.iteritems()}
# Ignore those items whose value is the same as the current value
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
if v != table.book_col_map.get(k, None)}
dirtied |= set(book_id_item_id_map)
# Update the book->col and col->book maps
deleted = set()
updated = {}
for book_id, item_ids in book_id_item_id_map.iteritems():
old_item_ids = table.book_col_map.get(book_id, None)
if old_item_ids:
for old_item_id in old_item_ids:
table.col_book_map[old_item_id].discard(book_id)
if item_ids:
table.book_col_map[book_id] = item_ids
for item_id in item_ids:
table.col_book_map[item_id].add(book_id)
updated[book_id] = item_ids
else:
table.book_col_map.pop(book_id, None)
deleted.add(book_id)
# Update the db link table
if deleted:
db.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
((k,) for k in deleted))
if updated:
vals = (
(book_id, val) for book_id, vals in updated.iteritems()
for val in vals
)
db.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
((k,) for k in updated))
db.executemany('INSERT INTO {0}(book,{1}) VALUES(?, ?)'.format(
table.link_table, m['link_column']), vals)
if is_authors:
aus_map = {book_id:field.author_sort_for_book(book_id) for book_id
in updated}
field.author_sort_field.writer.set_books(aus_map, db)
# Remove no longer used items
remove = {item_id for item_id in table.id_map if not
table.col_book_map.get(item_id, False)}
if remove:
db.executemany('DELETE FROM %s WHERE id=?'%m['table'],
((item_id,) for item_id in remove))
for item_id in remove:
del table.id_map[item_id]
table.col_book_map.pop(item_id, None)
if is_authors:
table.asort_map.pop(item_id, None)
table.alink_map.pop(item_id, None)
return dirtied
def icu_collator(s1, s2):
return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8'))
class _Interpreter(object):
def error(self, message):
m = 'Interpreter: ' + message
raise ValueError(m)
def program(self, funcs, parent, prog, val, is_call=False, args=None):
self.parent = parent
self.parent_kwargs = parent.kwargs
self.parent_book = parent.book
self.funcs = funcs
self.locals = {'$': val}
if is_call:
return self.do_node_call(CallNode(prog, None), args=args)
return self.expression_list(prog)
def expression_list(self, prog):
val = ''
for p in prog:
val = self.expr(p)
return val
INFIX_STRING_OPS = {
"==": lambda x, y: strcmp(x, y) == 0,
"!=": lambda x, y: strcmp(x, y) != 0,
"<": lambda x, y: strcmp(x, y) < 0,
"<=": lambda x, y: strcmp(x, y) <= 0,
">": lambda x, y: strcmp(x, y) > 0,
">=": lambda x, y: strcmp(x, y) >= 0,
}
def do_node_string_infix(self, prog):
try:
left = self.expr(prog.left)
right = self.expr(prog.right)
return ('1' if self.INFIX_STRING_OPS[prog.operator](left, right)
else '')
except:
self.error(
_('Error during string comparison. Operator {0}').format(
prog.operator))
INFIX_NUMERIC_OPS = {
"==#": lambda x, y: x == y,
"!=#": lambda x, y: x != y,
"<#": lambda x, y: x < y,
"<=#": lambda x, y: x <= y,
">#": lambda x, y: x > y,
">=#": lambda x, y: x >= y,
}
def float_deal_with_none(self, v):
# Undefined values and the string 'None' are assumed to be zero.
# The reason for string 'None': raw_field returns it for undefined values
return float(v if v and v != 'None' else 0)
def do_node_numeric_infix(self, prog):
try:
left = self.float_deal_with_none(self.expr(prog.left))
right = self.float_deal_with_none(self.expr(prog.right))
return '1' if self.INFIX_NUMERIC_OPS[prog.operator](left,
right) else ''
except:
self.error(
_('Value used in comparison is not a number. Operator {0}').
format(prog.operator))
def do_node_if(self, prog):
test_part = self.expr(prog.condition)
if test_part:
return self.expression_list(prog.then_part)
elif prog.else_part:
return self.expression_list(prog.else_part)
return ''
def do_node_rvalue(self, prog):
try:
return self.locals[prog.name]
except:
self.error(_('Unknown identifier {0}').format(prog.name))
def do_node_func(self, prog):
args = list()
for arg in prog.expression_list:
# evaluate the expression (recursive call)
args.append(self.expr(arg))
# Evaluate the function.
id_ = prog.name.strip()
cls = self.funcs[id_]
return cls.eval_(self.parent, self.parent_kwargs, self.parent_book,
self.locals, *args)
def do_node_call(self, prog, args=None):
if args is None:
args = []
for arg in prog.expression_list:
# evaluate the expression (recursive call)
args.append(self.expr(arg))
saved_locals = self.locals
self.locals = {}
for dex, v in enumerate(args):
self.locals['*arg_' + str(dex)] = v
val = self.expression_list(prog.function)
self.locals = saved_locals
return val
def do_node_arguments(self, prog):
for dex, arg in enumerate(prog.expression_list):
self.locals[arg.left] = self.locals.get('*arg_' + str(dex),
self.expr(arg.right))
return ''
def do_node_constant(self, prog):
return prog.value
def do_node_field(self, prog):
try:
name = self.expr(prog.expression)
try:
return self.parent.get_value(name, [], self.parent_kwargs)
except:
self.error(_('Unknown field {0}').format(name))
except ValueError as e:
raise e
except:
self.error(_('Unknown field {0}').format('internal parse error'))
def do_node_raw_field(self, prog):
try:
name = self.expr(prog.expression)
res = getattr(self.parent_book, name, None)
if res is not None:
if isinstance(res, list):
fm = self.parent_book.metadata_for_field(name)
if fm is None:
return ', '.join(res)
return fm['is_multiple']['list_to_ui'].join(res)
return unicode_type(res)
except ValueError as e:
raise e
except:
self.error(_('Unknown field {0}').format('internal parse error'))
def do_node_assign(self, prog):
t = self.expr(prog.right)
self.locals[prog.left] = t
return t
def do_node_first_non_empty(self, prog):
for expr in prog.expression_list:
if v := self.expr(expr):
return v
return ''
def icu_collator(s1, s2):
return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8'))
class _Interpreter(object):
def error(self, message, line_number):
m = _('Interpreter: {0} - line number {1}').format(message, line_number)
raise ValueError(m)
def program(self, funcs, parent, prog, val, is_call=False, args=None,
global_vars=None, break_reporter=None):
self.parent = parent
self.parent_kwargs = parent.kwargs
self.parent_book = parent.book
self.funcs = funcs
self.locals = {'$':val}
self.override_line_number = None
self.global_vars = global_vars if isinstance(global_vars, dict) else {}
if break_reporter:
self.break_reporter = self.call_break_reporter
self.real_break_reporter = break_reporter
else:
self.break_reporter = None
try:
if is_call:
ret = self.do_node_call(CallNode(1, prog, None), args=args)
else:
ret = self.expression_list(prog)
except ReturnExecuted as e:
ret = e.get_value()
return ret
def call_break_reporter(self, txt, val, line_number):
self.real_break_reporter(txt, val, self.locals,
self.override_line_number if self.override_line_number
else line_number)
def expression_list(self, prog):
val = ''
try:
for p in prog:
val = self.expr(p)
except (BreakExecuted, ContinueExecuted) as e:
e.set_value(val)
raise e
return val
INFIX_STRING_COMPARE_OPS = {
"==": lambda x, y: strcmp(x, y) == 0,
"!=": lambda x, y: strcmp(x, y) != 0,
"<": lambda x, y: strcmp(x, y) < 0,
"<=": lambda x, y: strcmp(x, y) <= 0,
">": lambda x, y: strcmp(x, y) > 0,
">=": lambda x, y: strcmp(x, y) >= 0,
"in": lambda x, y: re.search(x, y, flags=re.I),
"inlist": lambda x, y: list(filter(partial(re.search, x, flags=re.I),
[v.strip() for v in y.split(',') if v.strip()]))
}
def do_node_string_infix(self, prog):
try:
left = self.expr(prog.left)
right = self.expr(prog.right)
res = '1' if self.INFIX_STRING_COMPARE_OPS[prog.operator](left, right) else ''
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
except (StopException, ValueError) as e:
raise e
except:
self.error(_("Error during string comparison: "
"operator '{0}'").format(prog.operator), prog.line_number)
INFIX_NUMERIC_COMPARE_OPS = {
"==#": lambda x, y: x == y,
"!=#": lambda x, y: x != y,
"<#": lambda x, y: x < y,
"<=#": lambda x, y: x <= y,
">#": lambda x, y: x > y,
">=#": lambda x, y: x >= y,
}
def float_deal_with_none(self, v):
# Undefined values and the string 'None' are assumed to be zero.
# The reason for string 'None': raw_field returns it for undefined values
return float(v if v and v != 'None' else 0)
def do_node_numeric_infix(self, prog):
try:
left = self.float_deal_with_none(self.expr(prog.left))
right = self.float_deal_with_none(self.expr(prog.right))
res = '1' if self.INFIX_NUMERIC_COMPARE_OPS[prog.operator](left, right) else ''
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
except (StopException, ValueError) as e:
raise e
except:
self.error(_("Value used in comparison is not a number: "
"operator '{0}'").format(prog.operator), prog.line_number)
def do_node_if(self, prog):
line_number = prog.line_number
test_part = self.expr(prog.condition)
if self.break_reporter:
self.break_reporter("'if': condition value", test_part, line_number)
if test_part:
v = self.expression_list(prog.then_part)
if self.break_reporter:
self.break_reporter("'if': then-block value", v, line_number)
return v
elif prog.else_part:
v = self.expression_list(prog.else_part)
if self.break_reporter:
self.break_reporter("'if': else-block value", v, line_number)
return v
return ''
def do_node_rvalue(self, prog):
try:
if (self.break_reporter):
self.break_reporter(prog.node_name, self.locals[prog.name], prog.line_number)
return self.locals[prog.name]
except:
self.error(_("Unknown identifier '{0}'").format(prog.name), prog.line_number)
def do_node_func(self, prog):
args = list()
for arg in prog.expression_list:
# evaluate the expression (recursive call)
args.append(self.expr(arg))
# Evaluate the function.
id_ = prog.name.strip()
cls = self.funcs[id_]
res = cls.eval_(self.parent, self.parent_kwargs,
self.parent_book, self.locals, *args)
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
def do_node_call(self, prog, args=None):
if (self.break_reporter):
self.break_reporter(prog.node_name, _('before evaluating arguments'), prog.line_number)
if args is None:
args = []
for arg in prog.expression_list:
# evaluate the expression (recursive call)
args.append(self.expr(arg))
saved_locals = self.locals
self.locals = {}
for dex, v in enumerate(args):
self.locals['*arg_'+ str(dex)] = v
if (self.break_reporter):
self.break_reporter(prog.node_name, _('after evaluating arguments'), prog.line_number)
saved_line_number = self.override_line_number
self.override_line_number = (self.override_line_number if self.override_line_number
else prog.line_number)
try:
val = self.expression_list(prog.function)
except ReturnExecuted as e:
val = e.get_value()
self.override_line_number = saved_line_number
self.locals = saved_locals
if (self.break_reporter):
self.break_reporter(prog.node_name + _(' returned value'), val, prog.line_number)
return val
def do_node_arguments(self, prog):
for dex, arg in enumerate(prog.expression_list):
self.locals[arg.left] = self.locals.get('*arg_'+ str(dex), self.expr(arg.right))
if (self.break_reporter):
self.break_reporter(prog.node_name, '', prog.line_number)
return ''
def do_node_globals(self, prog):
res = ''
for arg in prog.expression_list:
res = self.locals[arg.left] = self.global_vars.get(arg.left, self.expr(arg.right))
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
def do_node_set_globals(self, prog):
res = ''
for arg in prog.expression_list:
res = self.global_vars[arg.left] = self.locals.get(arg.left, self.expr(arg.right))
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
def do_node_constant(self, prog):
if (self.break_reporter):
self.break_reporter(prog.node_name, prog.value, prog.line_number)
return prog.value
def do_node_field(self, prog):
try:
name = self.expr(prog.expression)
try:
res = self.parent.get_value(name, [], self.parent_kwargs)
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
except:
self.error(_("Unknown field '{0}'").format(name), prog.line_number)
except (StopException, ValueError) as e:
raise e
except:
self.error(_("Unknown field '{0}'").format('internal parse error'),
prog.line_number)
def do_node_raw_field(self, prog):
try:
name = self.expr(prog.expression)
res = getattr(self.parent_book, name, None)
if res is None and prog.default is not None:
res = self.expr(prog.default)
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
if res is not None:
if isinstance(res, list):
fm = self.parent_book.metadata_for_field(name)
if fm is None:
res = ', '.join(res)
else:
res = fm['is_multiple']['list_to_ui'].join(res)
else:
res = unicode_type(res)
else:
res = unicode_type(res) # Should be the string "None"
if (self.break_reporter):
self.break_reporter(prog.node_name, res, prog.line_number)
return res
except (StopException, ValueError) as e:
raise e
except:
self.error(_("Unknown field '{0}'").format('internal parse error'),
prog.line_number)
def do_node_assign(self, prog):
t = self.expr(prog.right)
self.locals[prog.left] = t
if (self.break_reporter):
self.break_reporter(prog.node_name, t, prog.line_number)
return t
def do_node_first_non_empty(self, prog):
for expr in prog.expression_list:
if v := self.expr(expr):
if (self.break_reporter):
self.break_reporter(prog.node_name, v, prog.line_number)
return v
if (self.break_reporter):
self.break_reporter(prog.node_name, '', prog.line_number)
return ''
class _Parser(object):
LEX_OP = 1
LEX_ID = 2
LEX_CONST = 3
LEX_EOF = 4
LEX_INFIX = 5
LEX_IF = 6
LEX_THEN = 7
LEX_ELSE = 8
LEX_FI = 9
def __init__(self, val, prog, funcs, parent):
self.lex_pos = 0
self.prog = prog[0]
self.prog_len = len(self.prog)
if prog[1] != '':
self.error(
_('Failed to scan program. Invalid input {0}').format(prog[1]))
self.parent = parent
self.parent_kwargs = parent.kwargs
self.parent_book = parent.book
self.locals = {'$': val}
self.funcs = funcs
def error(self, message):
try:
tval = "'" + self.prog[self.lex_pos - 1][1] + "'"
except:
tval = _('Unknown')
m = 'Formatter: ' + message + _(' near')
if self.lex_pos > 0:
m = '{0} {1}'.format(m, tval)
elif self.lex_pos < self.prog_len:
m = '{0} {1}'.format(m, tval)
else:
m = '{0} {1}'.format(m, _('end of program'))
raise ValueError(m)
def token(self):
try:
token = self.prog[self.lex_pos][1]
self.lex_pos += 1
return token
except:
return None
def consume(self):
self.lex_pos += 1
def token_op_is_equals(self):
try:
token = self.prog[self.lex_pos]
return token[1] == '=' and token[0] == self.LEX_OP
except:
return False
def token_op_is_infix_compare(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_INFIX
except:
return False
def token_op_is_lparen(self):
try:
token = self.prog[self.lex_pos]
return token[1] == '(' and token[0] == self.LEX_OP
except:
return False
def token_op_is_rparen(self):
try:
token = self.prog[self.lex_pos]
return token[1] == ')' and token[0] == self.LEX_OP
except:
return False
def token_op_is_comma(self):
try:
token = self.prog[self.lex_pos]
return token[1] == ',' and token[0] == self.LEX_OP
except:
return False
def token_op_is_semicolon(self):
try:
token = self.prog[self.lex_pos]
return token[1] == ';' and token[0] == self.LEX_OP
except:
return False
def token_is_id(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_ID
except:
return False
def token_is_if(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_IF
except:
return False
def token_is_then(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_THEN
except:
return False
def token_is_else(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_ELSE
except:
return False
def token_is_fi(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_FI
except:
return False
def token_is_constant(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_CONST
except:
return False
def token_is_eof(self):
try:
return self.prog[self.lex_pos][0] == self.LEX_EOF
except:
return True
def program(self):
val = self.statement()
if not self.token_is_eof():
self.error(_('Syntax error - program ends before EOF'))
return val
def statement(self):
val = ''
while not self.token_is_eof():
val = self.infix_expr()
if not self.token_op_is_semicolon():
break
self.consume()
return val
def consume_if(self):
self.consume()
while not self.token_is_fi():
if self.token_is_if():
self.consume_if()
self.consume()
def consume_then_branch(self):
while not (self.token_is_eof() or self.token_is_fi()
or self.token_is_else()):
if self.token_is_if():
self.consume_if()
self.consume()
def consume_else_branch(self):
while not (self.token_is_eof() or self.token_is_fi()):
if self.token_is_if():
self.consume_if()
self.consume()
def if_expression(self):
self.consume()
val = ''
test_part = self.infix_expr()
if not self.token_is_then():
self.error(_("Missing 'then' in if statement"))
if test_part:
self.consume()
val = self.statement()
if not (self.token_is_else() or self.token_is_fi()):
self.error(_("Missing 'else' or 'fi' in if statement"))
self.consume_else_branch()
else:
self.consume_then_branch()
if self.token_is_else():
self.consume()
val = self.statement()
if not self.token_is_fi():
self.error(_("Missing 'fi' in if statement"))
self.consume()
return val
INFIX_OPS = {
"==": lambda x, y: strcmp(x, y) == 0,
"!=": lambda x, y: strcmp(x, y) != 0,
"<": lambda x, y: strcmp(x, y) < 0,
"<=": lambda x, y: strcmp(x, y) <= 0,
">": lambda x, y: strcmp(x, y) > 0,
">=": lambda x, y: strcmp(x, y) >= 0,
"==#": lambda x, y: float(x) == float(y) if x and y else False,
"!=#": lambda x, y: float(x) != float(y) if x and y else False,
"<#": lambda x, y: float(x) < float(y) if x and y else False,
"<=#": lambda x, y: float(x) <= float(y) if x and y else False,
">#": lambda x, y: float(x) > float(y) if x and y else False,
">=#": lambda x, y: float(x) >= float(y) if x and y else False,
}
def infix_expr(self):
left = self.expr()
if self.token_op_is_infix_compare():
t = self.token()
right = self.expr()
return '1' if self.INFIX_OPS[t](left, right) else ''
return left
def expr(self):
if self.token_is_if():
return self.if_expression()
if self.token_is_id():
# We have an identifier. Determine if it is a function
id_ = self.token()
if not self.token_op_is_lparen():
if self.token_op_is_equals():
# classic assignment statement
self.consume()
cls = self.funcs['assign']
return cls.eval_(self.parent, self.parent_kwargs,
self.parent_book, self.locals, id_,
self.infix_expr())
val = self.locals.get(id_, None)
if val is None:
self.error(_('Unknown identifier {0}').format(id_))
return val
# We have a function.
# Check if it is a known one. We do this here so error reporting is
# better, as it can identify the tokens near the problem.
id_ = id_.strip()
if id_ not in self.funcs:
self.error(_('Unknown function {0}').format(id_))
# Eat the paren
self.consume()
args = list()
while not self.token_op_is_rparen():
if id_ == 'assign' and len(args) == 0:
# Must handle the lvalue semantics of the assign function.
# The first argument is the name of the destination, not
# the value.
if not self.token_is_id():
self.error(
_("'Assign' requires the first parameter be an id")
)
args.append(self.token())
else:
# evaluate the argument (recursive call)
args.append(self.infix_expr())
if not self.token_op_is_comma():
break
self.consume()
if self.token() != ')':
self.error(_('Missing closing parenthesis'))
# Evaluate the function.
if id_ == 'field':
# Evaluate the 'field' function inline for performance
if len(args) != 1:
self.error(
_('Incorrect number of arguments for function {0}').
format(id_))
return self.parent.get_value(args[0], [], self.parent_kwargs)
cls = self.funcs[id_]
if cls.arg_count != -1 and len(args) != cls.arg_count:
self.error(
_('Incorrect number of arguments for function {0}').format(
id_))
return cls.eval_(self.parent, self.parent_kwargs, self.parent_book,
self.locals, *args)
elif self.token_is_constant():
# String or number
return self.token()
else:
self.error(_('Expression is not function or constant'))
def update_state_and_val(self):
# Handle case change if the authors box changed
aus = authors_to_sort_string(self.authors_edit.current_val)
if strcmp(aus, self.current_val) == 0:
self.current_val = aus
self.update_state()
