def __str__(self):
res = u'DBOvl('
for k in sorted(self.d_assertz):
for clause in self.d_assertz[k]:
res += u'+' + limit_str(text_type(clause), 40)
for k in sorted(self.d_retracted):
for p in self.d_retracted[k]:
res += u'-' + limit_str(text_type(p), 40)
res += u')'
return res
def _trace_fn(self, label, env):
if not self.trace:
return
indent = ' '
logging.info(u"%s %s" % (indent, label))
# import pdb; pdb.set_trace()
for k in sorted(env):
logging.info(u"%s %s=%s" %
(indent, k, limit_str(repr(env[k]), 80)))
def _trace(self, label, goal):
if not self.trace:
return
# logging.debug ('label: %s, goal: %s' % (label, unicode(goal)))
depth = goal.get_depth()
# ind = depth * ' ' + len(label) * ' '
res = u'!' if goal.negate else u''
if goal.head:
res += limit_str(unicode(goal.head), 60)
else:
res += u'TOP'
res += ' '
for i, t in enumerate(goal.terms):
if i == goal.inx:
res += u" -> " + limit_str(unicode(t), 60)
res += ' [' + unicode(goal.location) + ']'
# indent = depth*' ' + len(label) * ' '
indent = depth * ' '
logging.info(u"%s %s: %s" % (indent, label, res))
for k in sorted(goal.env):
if k != ASSERT_OVERLAY_VAR_NAME:
logging.info(u"%s %s=%s" %
(indent, k, limit_str(repr(goal.env[k]), 100)))
if ASSERT_OVERLAY_VAR_NAME in goal.env:
goal.env[ASSERT_OVERLAY_VAR_NAME].log_trace(indent)
def _ner_learn(self, lang, cls):
entities = []
labels = []
for nerdata in self.session.query(model.NERData).filter(
model.NERData.lang == lang).filter(model.NERData.cls == cls):
entities.append(nerdata.entity)
labels.append(nerdata.label)
if not lang in self.ner_dict:
self.ner_dict[lang] = {}
if not cls in self.ner_dict[lang]:
self.ner_dict[lang][cls] = {}
nd = self.ner_dict[lang][cls]
for i, entity in enumerate(entities):
label = labels[i]
for j, token in enumerate(tokenize(label, lang=lang)):
if not token in nd:
nd[token] = {}
if not entity in nd[token]:
nd[token][entity] = set([])
nd[token][entity].add(j)
# logging.debug ('ner_learn: %4d %s %s: %s -> %s %s' % (i, entity, label, token, cls, lang))
cnt = 0
for token in nd:
# import pdb; pdb.set_trace()
# s1 = repr(nd[token])
# s2 = limit_str(s1, 10)
logging.debug('ner_learn: nd[%-20s]=%s' %
(token, misc.limit_str(repr(nd[token]), 80)))
cnt += 1
if cnt > 10:
break
def _ner_learn(lang, cls, entities, labels):
global ner_dict
# import pdb; pdb.set_trace()
if not lang in ner_dict:
ner_dict[lang] = {}
if not cls in ner_dict[lang]:
ner_dict[lang][cls] = {}
nd = ner_dict[lang][cls]
for i, entity in enumerate(entities):
label = labels[i]
for j, token in enumerate(tokenize(label, lang=lang)):
if not token in nd:
nd[token] = {}
if not entity in nd[token]:
nd[token][entity] = set([])
nd[token][entity].add(j)
# logging.debug ('ner_learn: %4d %s %s: %s -> %s %s' % (i, entity, label, token, cls, lang))
cnt = 0
for token in nd:
# import pdb; pdb.set_trace()
# s1 = repr(nd[token])
# s2 = limit_str(s1, 10)
logging.debug('ner_learn: nd[%-20s]=%s' %
(token, limit_str(repr(nd[token]), 80)))
cnt += 1
if cnt > 10:
break
return True
def _ner_learn(self, lang, cls):
entities = []
labels = []
for nerdata in self.session.query(model.NERData).filter(model.NERData.lang==lang).filter(model.NERData.cls==cls):
entities.append(nerdata.entity)
labels.append(nerdata.label)
if not lang in self.ner_dict:
self.ner_dict[lang] = {}
if not cls in self.ner_dict[lang]:
self.ner_dict[lang][cls] = {}
nd = self.ner_dict[lang][cls]
for i, entity in enumerate(entities):
label = labels[i]
for j, token in enumerate(tokenize(label, lang=lang)):
if not token in nd:
nd[token] = {}
if not entity in nd[token]:
nd[token][entity] = set([])
nd[token][entity].add(j)
# logging.debug ('ner_learn: %4d %s %s: %s -> %s %s' % (i, entity, label, token, cls, lang))
cnt = 0
for token in nd:
# import pdb; pdb.set_trace()
# s1 = repr(nd[token])
# s2 = limit_str(s1, 10)
logging.debug ('ner_learn: nd[%-20s]=%s' % (token, misc.limit_str(repr(nd[token]), 80)))
cnt += 1
if cnt > 10:
break
def log_trace(self, indent):
for k in sorted(self.d_assertz):
for clause in self.d_assertz[k]:
logging.info(u"%s [O] %s" %
(indent, limit_str(text_type(clause), 100)))
def test_limit_str(self):
self.assertEqual(misc.limit_str('1234567890', 10), '1234567890')
self.assertEqual(misc.limit_str('1234567890', 9), '123456...')