def test_sorting(self): " Test the various sorting APIs " german = """Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse""".split() german_good = ( """Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag""".split() ) french = """dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché""".split() french_good = """bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché""".split() # noqa # Test corner cases sort_key = icu.sort_key s = "\U0001f431" self.ae( sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), "UTF-8 encoded object not correctly decoded to generate sort key", ) self.ae(s.encode("utf-16"), s.encode("utf-16"), "Undecodable bytestring not returned as itself") self.ae(b"", sort_key(None)) self.ae(0, icu.strcmp(None, b"")) self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding()))) # Test locales with make_collation_func("dsk", "de", func="sort_key") as dsk: self.ae(german_good, sorted(german, key=dsk)) with make_collation_func("dcmp", "de", template="_strcmp_template") as dcmp: for x in german: for y in german: self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y)) with make_collation_func("fsk", "fr", func="sort_key") as fsk: self.ae(french_good, sorted(french, key=fsk)) with make_collation_func("fcmp", "fr", template="_strcmp_template") as fcmp: for x in french: for y in french: self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y)) with make_collation_func("ssk", "es", func="sort_key") as ssk: self.assertNotEqual(ssk("peña"), ssk("pena")) with make_collation_func("scmp", "es", template="_strcmp_template") as scmp: self.assertNotEqual(0, scmp("pena", "peña")) for k, v in {"pèché": "peche", "flüße": "Flusse", "Štepánek": "ŠtepaneK"}.iteritems(): self.ae(0, icu.primary_strcmp(k, v)) # Test different types of collation self.ae(icu.primary_sort_key("Aä"), icu.primary_sort_key("aa")) self.assertLess(icu.numeric_sort_key("something 2"), icu.numeric_sort_key("something 11")) self.assertLess(icu.case_sensitive_sort_key("A"), icu.case_sensitive_sort_key("a")) self.ae(0, icu.strcmp("a", "A")) self.ae(cmp("a", "A"), icu.case_sensitive_strcmp("a", "A")) self.ae(0, icu.primary_strcmp("ä", "A"))
def add_category(self): self.save_category() cat_name = unicode_type(self.input_box.text()).strip() if cat_name == '': return False comps = [c.strip() for c in cat_name.split('.') if c.strip()] if len(comps) == 0 or '.'.join(comps) != cat_name: error_dialog(self, _('Invalid name'), _('That name contains leading or trailing periods, ' 'multiple periods in a row or spaces before ' 'or after periods.')).exec_() return False for c in sorted(self.categories.keys(), key=sort_key): if strcmp(c, cat_name) == 0 or \ (icu_lower(cat_name).startswith(icu_lower(c) + '.') and not cat_name.startswith(c + '.')): error_dialog(self, _('Name already used'), _('That name is already used, perhaps with different case.')).exec_() return False if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name self.categories[cat_name] = [] self.applied_items = [] self.populate_category_list() self.input_box.clear() self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True
def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt): v = strcmp(x, y) if v < 0: return lt if v == 0: return eq return gt
def add_category(self): self.save_category() cat_name = str(self.input_box.text()).strip() if cat_name == '': return False comps = [c.strip() for c in cat_name.split('.') if c.strip()] if len(comps) == 0 or '.'.join(comps) != cat_name: error_dialog( self, _('Invalid name'), _('That name contains leading or trailing periods, ' 'multiple periods in a row or spaces before ' 'or after periods.')).exec_() return False for c in sorted(list(self.categories.keys()), key=sort_key): if strcmp(c, cat_name) == 0 or \ (icu_lower(cat_name).startswith(icu_lower(c) + '.') and not cat_name.startswith(c + '.')): error_dialog( self, _('Name already used'), _('That name is already used, perhaps with different case.' )).exec_() return False if cat_name not in self.categories: self.category_box.clear() self.current_cat_name = cat_name self.categories[cat_name] = [] self.applied_items = [] self.populate_category_list() self.input_box.clear() self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True
def rename_category(self): self.save_category() cat_name = str(self.input_box.text()).strip() if cat_name == '': return False if not self.current_cat_name: return False comps = [c.strip() for c in cat_name.split('.') if c.strip()] if len(comps) == 0 or '.'.join(comps) != cat_name: error_dialog( self, _('Invalid name'), _('That name contains leading or trailing periods, ' 'multiple periods in a row or spaces before ' 'or after periods.')).exec_() return False for c in self.categories: if strcmp(c, cat_name) == 0: error_dialog( self, _('Name already used'), _('That name is already used, perhaps with different case.' )).exec_() return False # The order below is important because of signals self.categories[cat_name] = self.categories[self.current_cat_name] del self.categories[self.current_cat_name] self.current_cat_name = None self.populate_category_list() self.input_box.clear() self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True
def rename_category(self): self.save_category() cat_name = unicode_type(self.input_box.text()).strip() if cat_name == '': return False if not self.current_cat_name: return False comps = [c.strip() for c in cat_name.split('.') if c.strip()] if len(comps) == 0 or '.'.join(comps) != cat_name: error_dialog(self, _('Invalid name'), _('That name contains leading or trailing periods, ' 'multiple periods in a row or spaces before ' 'or after periods.')).exec_() return False for c in self.categories: if strcmp(c, cat_name) == 0: error_dialog(self, _('Name already used'), _('That name is already used, perhaps with different case.')).exec_() return False # The order below is important because of signals self.categories[cat_name] = self.categories[self.current_cat_name] del self.categories[self.current_cat_name] self.current_cat_name = None self.populate_category_list() self.input_box.clear() self.category_box.setCurrentIndex(self.category_box.findText(cat_name)) return True
def test_sorting(self): ' Test the various sorting APIs ' german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split() german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split() french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split() french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split() # noqa # Test corner cases sort_key = icu.sort_key s = '\U0001f431' self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key') self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself') self.ae(b'', sort_key(None)) self.ae(0, icu.strcmp(None, b'')) self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding()))) # Test locales with make_collation_func('dsk', 'de', func='sort_key') as dsk: self.ae(german_good, sorted(german, key=dsk)) with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp: for x in german: for y in german: self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y)) with make_collation_func('fsk', 'fr', func='sort_key') as fsk: self.ae(french_good, sorted(french, key=fsk)) with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp: for x in french: for y in french: self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y)) with make_collation_func('ssk', 'es', func='sort_key') as ssk: self.assertNotEqual(ssk('peña'), ssk('pena')) with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp: self.assertNotEqual(0, scmp('pena', 'peña')) for k, v in iteritems({u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}): self.ae(0, icu.primary_strcmp(k, v)) # Test different types of collation self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa')) self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11')) self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a')) self.ae(0, icu.strcmp('a', 'A')) self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A')) self.ae(0, icu.primary_strcmp('ä', 'A'))
def test_sorting(self): ' Test the various sorting APIs ' german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split() german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split() french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split() french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split() # noqa # Test corner cases sort_key = icu.sort_key s = '\U0001f431' self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key') self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself') self.ae(b'', sort_key(None)) self.ae(0, icu.strcmp(None, b'')) self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding()))) # Test locales with make_collation_func('dsk', 'de', func='sort_key') as dsk: self.ae(german_good, sorted(german, key=dsk)) with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp: for x in german: for y in german: self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y)) with make_collation_func('fsk', 'fr', func='sort_key') as fsk: self.ae(french_good, sorted(french, key=fsk)) with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp: for x in french: for y in french: self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y)) with make_collation_func('ssk', 'es', func='sort_key') as ssk: self.assertNotEqual(ssk('peña'), ssk('pena')) with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp: self.assertNotEqual(0, scmp('pena', 'peña')) for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems(): self.ae(0, icu.primary_strcmp(k, v)) # Test different types of collation self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa')) self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11')) self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a')) self.ae(0, icu.strcmp('a', 'A')) self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A')) self.ae(0, icu.primary_strcmp('ä', 'A'))
def evaluate(self, formatter, kwargs, mi, locals, val, sep, str, fv, nfv): l = [v.strip() for v in val.split(sep) if v.strip()] c = [v.strip() for v in str.split(sep) if v.strip()] if l: for v in l: for t in c: if strcmp(t, v) == 0: return fv return nfv
def update_state(self, *args): au = unicode(self.authors_edit.text()) au = re.sub(r'\s+et al\.$', '', au) au = self.db.author_sort_from_authors(string_to_authors(au)) normal = strcmp(au, self.current_val) == 0 if normal: col = 'rgb(0, 255, 0, 20%)' else: col = 'rgb(255, 0, 0, 20%)' self.setStyleSheet('QLineEdit { color: black; ' 'background-color: %s; }' % col) tt = self.tooltips[0 if normal else 1] self.setToolTip(tt) self.setWhatsThis(tt)
def update_state(self, *args): au = unicode(self.authors_edit.text()) au = re.sub(r'\s+et al\.$', '', au) au = self.db.author_sort_from_authors(string_to_authors(au)) normal = strcmp(au, self.current_val) == 0 if normal: col = 'rgb(0, 255, 0, 20%)' else: col = 'rgb(255, 0, 0, 20%)' self.setStyleSheet('QLineEdit { color: black; ' 'background-color: %s; }'%col) tt = self.tooltips[0 if normal else 1] self.setToolTip(tt) self.setWhatsThis(tt)
def none_cmp(xx, yy): x = xx[1] y = yy[1] if x is None and y is None: # No sort_key needed here, because defaults are ascii return cmp(xx[2], yy[2]) if x is None: return 1 if y is None: return -1 if isinstance(x, basestring) and isinstance(y, basestring): c = strcmp(force_unicode(x), force_unicode(y)) else: c = cmp(x, y) if c != 0: return c # same as above -- no sort_key needed here return cmp(xx[2], yy[2])
def many_many(book_id_val_map, db, field, allow_case_change, *args): dirtied = set() m = field.metadata table = field.table dt = m['datatype'] is_authors = field.name == 'authors' # Map values to db ids, including any new values kmap = safe_lower if dt == 'text' else lambda x:x rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()} if len(rid_map) != len(table.id_map): # table has some entries that differ only in case, fix it table.fix_case_duplicates(db) rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()} val_map = {} case_changes = {} book_id_val_map = {k:uniq(vals, kmap) for k, vals in book_id_val_map.iteritems()} for vals in book_id_val_map.itervalues(): for val in vals: get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, case_changes, val_map, is_authors=is_authors) if case_changes: change_case(case_changes, dirtied, db, table, m, is_authors=is_authors) if is_authors: for item_id, val in case_changes.iteritems(): for book_id in table.col_book_map[item_id]: current_sort = field.db_author_sort_for_book(book_id) new_sort = field.author_sort_for_book(book_id) if strcmp(current_sort, new_sort) == 0: # The sort strings differ only by case, update the db # sort field.author_sort_field.writer.set_books({book_id:new_sort}, db) book_id_item_id_map = {k:tuple(val_map[v] for v in vals) for k, vals in book_id_val_map.iteritems()} # Ignore those items whose value is the same as the current value book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems() if v != table.book_col_map.get(k, None)} dirtied |= set(book_id_item_id_map) # Update the book->col and col->book maps deleted = set() updated = {} for book_id, item_ids in book_id_item_id_map.iteritems(): old_item_ids = table.book_col_map.get(book_id, None) if old_item_ids: for old_item_id in old_item_ids: table.col_book_map[old_item_id].discard(book_id) if item_ids: table.book_col_map[book_id] = item_ids for item_id in item_ids: table.col_book_map[item_id].add(book_id) updated[book_id] = item_ids else: table.book_col_map.pop(book_id, None) deleted.add(book_id) # Update the db link table if deleted: db.executemany('DELETE FROM %s WHERE book=?'%table.link_table, ((k,) for k in deleted)) if updated: vals = ( (book_id, val) for book_id, vals in updated.iteritems() for val in vals ) db.executemany('DELETE FROM %s WHERE book=?'%table.link_table, ((k,) for k in updated)) db.executemany('INSERT INTO {0}(book,{1}) VALUES(?, ?)'.format( table.link_table, m['link_column']), vals) if is_authors: aus_map = {book_id:field.author_sort_for_book(book_id) for book_id in updated} field.author_sort_field.writer.set_books(aus_map, db) # Remove no longer used items remove = {item_id for item_id in table.id_map if not table.col_book_map.get(item_id, False)} if remove: db.executemany('DELETE FROM %s WHERE id=?'%m['table'], ((item_id,) for item_id in remove)) for item_id in remove: del table.id_map[item_id] table.col_book_map.pop(item_id, None) if is_authors: table.asort_map.pop(item_id, None) table.alink_map.pop(item_id, None) return dirtied
def update_state_and_val(self): # Handle case change if the authors box changed aus = authors_to_sort_string(self.authors_edit.current_val) if strcmp(aus, self.current_val) == 0: self.current_val = aus self.update_state()
class _Interpreter(object): def error(self, message): m = 'Interpreter: ' + message raise ValueError(m) def program(self, funcs, parent, prog, val): self.parent = parent self.parent_kwargs = parent.kwargs self.parent_book = parent.book self.funcs = funcs self.locals = {'$': val} return self.expression_list(prog) def expression_list(self, prog): val = '' for p in prog: val = self.expr(p) return val INFIX_STRING_OPS = { "==": lambda x, y: strcmp(x, y) == 0, "!=": lambda x, y: strcmp(x, y) != 0, "<": lambda x, y: strcmp(x, y) < 0, "<=": lambda x, y: strcmp(x, y) <= 0, ">": lambda x, y: strcmp(x, y) > 0, ">=": lambda x, y: strcmp(x, y) >= 0, } def do_node_string_infix(self, prog): try: left = self.expr(prog.left) right = self.expr(prog.right) return ('1' if self.INFIX_STRING_OPS[prog.operator](left, right) else '') except: self.error( _('Error during string comparison. Operator {0}').format( prog.operator)) INFIX_NUMERIC_OPS = { "==#": lambda x, y: x == y, "!=#": lambda x, y: x != y, "<#": lambda x, y: x < y, "<=#": lambda x, y: x <= y, ">#": lambda x, y: x > y, ">=#": lambda x, y: x >= y, } def float_deal_with_none(self, v): # Undefined values and the string 'None' are assumed to be zero. # The reason for string 'None': raw_field returns it for undefined values return float(v if v and v != 'None' else 0) def do_node_numeric_infix(self, prog): try: left = self.float_deal_with_none(self.expr(prog.left)) right = self.float_deal_with_none(self.expr(prog.right)) return '1' if self.INFIX_NUMERIC_OPS[prog.operator](left, right) else '' except: self.error( _('Value used in comparison is not a number. Operator {0}'). format(prog.operator)) def do_node_if(self, prog): test_part = self.expr(prog.condition) if test_part: return self.expression_list(prog.then_part) elif prog.else_part: return self.expression_list(prog.else_part) return '' def do_node_rvalue(self, prog): try: return self.locals[prog.name] except: self.error(_('Unknown identifier {0}').format(prog.name)) def do_node_func(self, prog): args = list() for arg in prog.expression_list: # evaluate the expression (recursive call) args.append(self.expr(arg)) # Evaluate the function. id_ = prog.name.strip() cls = self.funcs[id_] return cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, *args) def do_node_constant(self, prog): return prog.value def do_node_field(self, prog): try: name = self.expr(prog.expression) try: return self.parent.get_value(name, [], self.parent_kwargs) except: self.error(_('Unknown field {0}').format(name)) except ValueError as e: raise e except: self.error(_('Unknown field {0}').format('parse error')) def do_node_raw_field(self, prog): try: name = self.expr(prog.expression) res = getattr(self.parent_book, name, None) if res is not None: if isinstance(res, list): fm = self.parent_book.metadata_for_field(name) if fm is None: return ', '.join(res) return fm['is_multiple']['list_to_ui'].join(res) return unicode_type(res) except ValueError as e: raise e except: self.error(_('Unknown field {0}').format('parse error')) def do_node_assign(self, prog): t = self.expr(prog.right) self.locals[prog.left] = t return t NODE_OPS = { Node.NODE_IF: do_node_if, Node.NODE_ASSIGN: do_node_assign, Node.NODE_CONSTANT: do_node_constant, Node.NODE_RVALUE: do_node_rvalue, Node.NODE_FUNC: do_node_func, Node.NODE_FIELD: do_node_field, Node.NODE_RAW_FIELD: do_node_raw_field, Node.NODE_STRING_INFIX: do_node_string_infix, Node.NODE_NUMERIC_INFIX: do_node_numeric_infix, } def expr(self, prog): try: return self.NODE_OPS[prog.node_type](self, prog) except ValueError as e: raise e except: if (DEBUG): traceback.print_exc() self.error(_('Internal error evaluating an expression'))
def icu_collator(s1, s2): return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8'))
class _Interpreter(object): def error(self, message): m = 'Interpreter: ' + message raise ValueError(m) def program(self, funcs, parent, prog, val, is_call=False, args=None): self.parent = parent self.parent_kwargs = parent.kwargs self.parent_book = parent.book self.funcs = funcs self.locals = {'$': val} if is_call: return self.do_node_call(CallNode(prog, None), args=args) return self.expression_list(prog) def expression_list(self, prog): val = '' for p in prog: val = self.expr(p) return val INFIX_STRING_OPS = { "==": lambda x, y: strcmp(x, y) == 0, "!=": lambda x, y: strcmp(x, y) != 0, "<": lambda x, y: strcmp(x, y) < 0, "<=": lambda x, y: strcmp(x, y) <= 0, ">": lambda x, y: strcmp(x, y) > 0, ">=": lambda x, y: strcmp(x, y) >= 0, } def do_node_string_infix(self, prog): try: left = self.expr(prog.left) right = self.expr(prog.right) return ('1' if self.INFIX_STRING_OPS[prog.operator](left, right) else '') except: self.error( _('Error during string comparison. Operator {0}').format( prog.operator)) INFIX_NUMERIC_OPS = { "==#": lambda x, y: x == y, "!=#": lambda x, y: x != y, "<#": lambda x, y: x < y, "<=#": lambda x, y: x <= y, ">#": lambda x, y: x > y, ">=#": lambda x, y: x >= y, } def float_deal_with_none(self, v): # Undefined values and the string 'None' are assumed to be zero. # The reason for string 'None': raw_field returns it for undefined values return float(v if v and v != 'None' else 0) def do_node_numeric_infix(self, prog): try: left = self.float_deal_with_none(self.expr(prog.left)) right = self.float_deal_with_none(self.expr(prog.right)) return '1' if self.INFIX_NUMERIC_OPS[prog.operator](left, right) else '' except: self.error( _('Value used in comparison is not a number. Operator {0}'). format(prog.operator)) def do_node_if(self, prog): test_part = self.expr(prog.condition) if test_part: return self.expression_list(prog.then_part) elif prog.else_part: return self.expression_list(prog.else_part) return '' def do_node_rvalue(self, prog): try: return self.locals[prog.name] except: self.error(_('Unknown identifier {0}').format(prog.name)) def do_node_func(self, prog): args = list() for arg in prog.expression_list: # evaluate the expression (recursive call) args.append(self.expr(arg)) # Evaluate the function. id_ = prog.name.strip() cls = self.funcs[id_] return cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, *args) def do_node_call(self, prog, args=None): if args is None: args = [] for arg in prog.expression_list: # evaluate the expression (recursive call) args.append(self.expr(arg)) saved_locals = self.locals self.locals = {} for dex, v in enumerate(args): self.locals['*arg_' + str(dex)] = v val = self.expression_list(prog.function) self.locals = saved_locals return val def do_node_arguments(self, prog): for dex, arg in enumerate(prog.expression_list): self.locals[arg.left] = self.locals.get('*arg_' + str(dex), self.expr(arg.right)) return '' def do_node_constant(self, prog): return prog.value def do_node_field(self, prog): try: name = self.expr(prog.expression) try: return self.parent.get_value(name, [], self.parent_kwargs) except: self.error(_('Unknown field {0}').format(name)) except ValueError as e: raise e except: self.error(_('Unknown field {0}').format('internal parse error')) def do_node_raw_field(self, prog): try: name = self.expr(prog.expression) res = getattr(self.parent_book, name, None) if res is not None: if isinstance(res, list): fm = self.parent_book.metadata_for_field(name) if fm is None: return ', '.join(res) return fm['is_multiple']['list_to_ui'].join(res) return unicode_type(res) except ValueError as e: raise e except: self.error(_('Unknown field {0}').format('internal parse error')) def do_node_assign(self, prog): t = self.expr(prog.right) self.locals[prog.left] = t return t def do_node_first_non_empty(self, prog): for expr in prog.expression_list: if v := self.expr(expr): return v return ''
class _Interpreter(object): def error(self, message, line_number): m = _('Interpreter: {0} - line number {1}').format(message, line_number) raise ValueError(m) def program(self, funcs, parent, prog, val, is_call=False, args=None, global_vars=None, break_reporter=None): self.parent = parent self.parent_kwargs = parent.kwargs self.parent_book = parent.book self.funcs = funcs self.locals = {'$':val} self.override_line_number = None self.global_vars = global_vars if isinstance(global_vars, dict) else {} if break_reporter: self.break_reporter = self.call_break_reporter self.real_break_reporter = break_reporter else: self.break_reporter = None try: if is_call: ret = self.do_node_call(CallNode(1, prog, None), args=args) else: ret = self.expression_list(prog) except ReturnExecuted as e: ret = e.get_value() return ret def call_break_reporter(self, txt, val, line_number): self.real_break_reporter(txt, val, self.locals, self.override_line_number if self.override_line_number else line_number) def expression_list(self, prog): val = '' try: for p in prog: val = self.expr(p) except (BreakExecuted, ContinueExecuted) as e: e.set_value(val) raise e return val INFIX_STRING_COMPARE_OPS = { "==": lambda x, y: strcmp(x, y) == 0, "!=": lambda x, y: strcmp(x, y) != 0, "<": lambda x, y: strcmp(x, y) < 0, "<=": lambda x, y: strcmp(x, y) <= 0, ">": lambda x, y: strcmp(x, y) > 0, ">=": lambda x, y: strcmp(x, y) >= 0, "in": lambda x, y: re.search(x, y, flags=re.I), "inlist": lambda x, y: list(filter(partial(re.search, x, flags=re.I), [v.strip() for v in y.split(',') if v.strip()])) } def do_node_string_infix(self, prog): try: left = self.expr(prog.left) right = self.expr(prog.right) res = '1' if self.INFIX_STRING_COMPARE_OPS[prog.operator](left, right) else '' if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res except (StopException, ValueError) as e: raise e except: self.error(_("Error during string comparison: " "operator '{0}'").format(prog.operator), prog.line_number) INFIX_NUMERIC_COMPARE_OPS = { "==#": lambda x, y: x == y, "!=#": lambda x, y: x != y, "<#": lambda x, y: x < y, "<=#": lambda x, y: x <= y, ">#": lambda x, y: x > y, ">=#": lambda x, y: x >= y, } def float_deal_with_none(self, v): # Undefined values and the string 'None' are assumed to be zero. # The reason for string 'None': raw_field returns it for undefined values return float(v if v and v != 'None' else 0) def do_node_numeric_infix(self, prog): try: left = self.float_deal_with_none(self.expr(prog.left)) right = self.float_deal_with_none(self.expr(prog.right)) res = '1' if self.INFIX_NUMERIC_COMPARE_OPS[prog.operator](left, right) else '' if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res except (StopException, ValueError) as e: raise e except: self.error(_("Value used in comparison is not a number: " "operator '{0}'").format(prog.operator), prog.line_number) def do_node_if(self, prog): line_number = prog.line_number test_part = self.expr(prog.condition) if self.break_reporter: self.break_reporter("'if': condition value", test_part, line_number) if test_part: v = self.expression_list(prog.then_part) if self.break_reporter: self.break_reporter("'if': then-block value", v, line_number) return v elif prog.else_part: v = self.expression_list(prog.else_part) if self.break_reporter: self.break_reporter("'if': else-block value", v, line_number) return v return '' def do_node_rvalue(self, prog): try: if (self.break_reporter): self.break_reporter(prog.node_name, self.locals[prog.name], prog.line_number) return self.locals[prog.name] except: self.error(_("Unknown identifier '{0}'").format(prog.name), prog.line_number) def do_node_func(self, prog): args = list() for arg in prog.expression_list: # evaluate the expression (recursive call) args.append(self.expr(arg)) # Evaluate the function. id_ = prog.name.strip() cls = self.funcs[id_] res = cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, *args) if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res def do_node_call(self, prog, args=None): if (self.break_reporter): self.break_reporter(prog.node_name, _('before evaluating arguments'), prog.line_number) if args is None: args = [] for arg in prog.expression_list: # evaluate the expression (recursive call) args.append(self.expr(arg)) saved_locals = self.locals self.locals = {} for dex, v in enumerate(args): self.locals['*arg_'+ str(dex)] = v if (self.break_reporter): self.break_reporter(prog.node_name, _('after evaluating arguments'), prog.line_number) saved_line_number = self.override_line_number self.override_line_number = (self.override_line_number if self.override_line_number else prog.line_number) try: val = self.expression_list(prog.function) except ReturnExecuted as e: val = e.get_value() self.override_line_number = saved_line_number self.locals = saved_locals if (self.break_reporter): self.break_reporter(prog.node_name + _(' returned value'), val, prog.line_number) return val def do_node_arguments(self, prog): for dex, arg in enumerate(prog.expression_list): self.locals[arg.left] = self.locals.get('*arg_'+ str(dex), self.expr(arg.right)) if (self.break_reporter): self.break_reporter(prog.node_name, '', prog.line_number) return '' def do_node_globals(self, prog): res = '' for arg in prog.expression_list: res = self.locals[arg.left] = self.global_vars.get(arg.left, self.expr(arg.right)) if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res def do_node_set_globals(self, prog): res = '' for arg in prog.expression_list: res = self.global_vars[arg.left] = self.locals.get(arg.left, self.expr(arg.right)) if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res def do_node_constant(self, prog): if (self.break_reporter): self.break_reporter(prog.node_name, prog.value, prog.line_number) return prog.value def do_node_field(self, prog): try: name = self.expr(prog.expression) try: res = self.parent.get_value(name, [], self.parent_kwargs) if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res except: self.error(_("Unknown field '{0}'").format(name), prog.line_number) except (StopException, ValueError) as e: raise e except: self.error(_("Unknown field '{0}'").format('internal parse error'), prog.line_number) def do_node_raw_field(self, prog): try: name = self.expr(prog.expression) res = getattr(self.parent_book, name, None) if res is None and prog.default is not None: res = self.expr(prog.default) if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res if res is not None: if isinstance(res, list): fm = self.parent_book.metadata_for_field(name) if fm is None: res = ', '.join(res) else: res = fm['is_multiple']['list_to_ui'].join(res) else: res = unicode_type(res) else: res = unicode_type(res) # Should be the string "None" if (self.break_reporter): self.break_reporter(prog.node_name, res, prog.line_number) return res except (StopException, ValueError) as e: raise e except: self.error(_("Unknown field '{0}'").format('internal parse error'), prog.line_number) def do_node_assign(self, prog): t = self.expr(prog.right) self.locals[prog.left] = t if (self.break_reporter): self.break_reporter(prog.node_name, t, prog.line_number) return t def do_node_first_non_empty(self, prog): for expr in prog.expression_list: if v := self.expr(expr): if (self.break_reporter): self.break_reporter(prog.node_name, v, prog.line_number) return v if (self.break_reporter): self.break_reporter(prog.node_name, '', prog.line_number) return ''
class _Parser(object): LEX_OP = 1 LEX_ID = 2 LEX_CONST = 3 LEX_EOF = 4 LEX_INFIX = 5 LEX_IF = 6 LEX_THEN = 7 LEX_ELSE = 8 LEX_FI = 9 def __init__(self, val, prog, funcs, parent): self.lex_pos = 0 self.prog = prog[0] self.prog_len = len(self.prog) if prog[1] != '': self.error( _('Failed to scan program. Invalid input {0}').format(prog[1])) self.parent = parent self.parent_kwargs = parent.kwargs self.parent_book = parent.book self.locals = {'$': val} self.funcs = funcs def error(self, message): try: tval = "'" + self.prog[self.lex_pos - 1][1] + "'" except: tval = _('Unknown') m = 'Formatter: ' + message + _(' near') if self.lex_pos > 0: m = '{0} {1}'.format(m, tval) elif self.lex_pos < self.prog_len: m = '{0} {1}'.format(m, tval) else: m = '{0} {1}'.format(m, _('end of program')) raise ValueError(m) def token(self): try: token = self.prog[self.lex_pos][1] self.lex_pos += 1 return token except: return None def consume(self): self.lex_pos += 1 def token_op_is_equals(self): try: token = self.prog[self.lex_pos] return token[1] == '=' and token[0] == self.LEX_OP except: return False def token_op_is_infix_compare(self): try: return self.prog[self.lex_pos][0] == self.LEX_INFIX except: return False def token_op_is_lparen(self): try: token = self.prog[self.lex_pos] return token[1] == '(' and token[0] == self.LEX_OP except: return False def token_op_is_rparen(self): try: token = self.prog[self.lex_pos] return token[1] == ')' and token[0] == self.LEX_OP except: return False def token_op_is_comma(self): try: token = self.prog[self.lex_pos] return token[1] == ',' and token[0] == self.LEX_OP except: return False def token_op_is_semicolon(self): try: token = self.prog[self.lex_pos] return token[1] == ';' and token[0] == self.LEX_OP except: return False def token_is_id(self): try: return self.prog[self.lex_pos][0] == self.LEX_ID except: return False def token_is_if(self): try: return self.prog[self.lex_pos][0] == self.LEX_IF except: return False def token_is_then(self): try: return self.prog[self.lex_pos][0] == self.LEX_THEN except: return False def token_is_else(self): try: return self.prog[self.lex_pos][0] == self.LEX_ELSE except: return False def token_is_fi(self): try: return self.prog[self.lex_pos][0] == self.LEX_FI except: return False def token_is_constant(self): try: return self.prog[self.lex_pos][0] == self.LEX_CONST except: return False def token_is_eof(self): try: return self.prog[self.lex_pos][0] == self.LEX_EOF except: return True def program(self): val = self.statement() if not self.token_is_eof(): self.error(_('Syntax error - program ends before EOF')) return val def statement(self): val = '' while not self.token_is_eof(): val = self.infix_expr() if not self.token_op_is_semicolon(): break self.consume() return val def consume_if(self): self.consume() while not self.token_is_fi(): if self.token_is_if(): self.consume_if() self.consume() def consume_then_branch(self): while not (self.token_is_eof() or self.token_is_fi() or self.token_is_else()): if self.token_is_if(): self.consume_if() self.consume() def consume_else_branch(self): while not (self.token_is_eof() or self.token_is_fi()): if self.token_is_if(): self.consume_if() self.consume() def if_expression(self): self.consume() val = '' test_part = self.infix_expr() if not self.token_is_then(): self.error(_("Missing 'then' in if statement")) if test_part: self.consume() val = self.statement() if not (self.token_is_else() or self.token_is_fi()): self.error(_("Missing 'else' or 'fi' in if statement")) self.consume_else_branch() else: self.consume_then_branch() if self.token_is_else(): self.consume() val = self.statement() if not self.token_is_fi(): self.error(_("Missing 'fi' in if statement")) self.consume() return val INFIX_OPS = { "==": lambda x, y: strcmp(x, y) == 0, "!=": lambda x, y: strcmp(x, y) != 0, "<": lambda x, y: strcmp(x, y) < 0, "<=": lambda x, y: strcmp(x, y) <= 0, ">": lambda x, y: strcmp(x, y) > 0, ">=": lambda x, y: strcmp(x, y) >= 0, "==#": lambda x, y: float(x) == float(y) if x and y else False, "!=#": lambda x, y: float(x) != float(y) if x and y else False, "<#": lambda x, y: float(x) < float(y) if x and y else False, "<=#": lambda x, y: float(x) <= float(y) if x and y else False, ">#": lambda x, y: float(x) > float(y) if x and y else False, ">=#": lambda x, y: float(x) >= float(y) if x and y else False, } def infix_expr(self): left = self.expr() if self.token_op_is_infix_compare(): t = self.token() right = self.expr() return '1' if self.INFIX_OPS[t](left, right) else '' return left def expr(self): if self.token_is_if(): return self.if_expression() if self.token_is_id(): # We have an identifier. Determine if it is a function id_ = self.token() if not self.token_op_is_lparen(): if self.token_op_is_equals(): # classic assignment statement self.consume() cls = self.funcs['assign'] return cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, id_, self.infix_expr()) val = self.locals.get(id_, None) if val is None: self.error(_('Unknown identifier {0}').format(id_)) return val # We have a function. # Check if it is a known one. We do this here so error reporting is # better, as it can identify the tokens near the problem. id_ = id_.strip() if id_ not in self.funcs: self.error(_('Unknown function {0}').format(id_)) # Eat the paren self.consume() args = list() while not self.token_op_is_rparen(): if id_ == 'assign' and len(args) == 0: # Must handle the lvalue semantics of the assign function. # The first argument is the name of the destination, not # the value. if not self.token_is_id(): self.error( _("'Assign' requires the first parameter be an id") ) args.append(self.token()) else: # evaluate the argument (recursive call) args.append(self.infix_expr()) if not self.token_op_is_comma(): break self.consume() if self.token() != ')': self.error(_('Missing closing parenthesis')) # Evaluate the function. if id_ == 'field': # Evaluate the 'field' function inline for performance if len(args) != 1: self.error( _('Incorrect number of arguments for function {0}'). format(id_)) return self.parent.get_value(args[0], [], self.parent_kwargs) cls = self.funcs[id_] if cls.arg_count != -1 and len(args) != cls.arg_count: self.error( _('Incorrect number of arguments for function {0}').format( id_)) return cls.eval_(self.parent, self.parent_kwargs, self.parent_book, self.locals, *args) elif self.token_is_constant(): # String or number return self.token() else: self.error(_('Expression is not function or constant'))