def test_merge2(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: No assertions
DB2: No assertions
'''
print "Running test 2: "
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
#testmerge_display(db1, db2)
'''
Merge the two dbs
'''
merge(db1, db2)
'''
Check post-merge elements, make sure they match
'''
testmerge_check(db1, db2)
print "Finished test 2."
def test_merge1(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: No assertions
DB2: Assertions 0-9
'''
print "Running test 1: "
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a1 = Assertion.make(
'/data/test', '/rel/IsA',
['/test/assertion', 'test/test%d' % i])
a1.add_support(['/data/test/contributor/nholm'])
#testmerge_display(db1, db2)
'''
Merge the two dbs
'''
merge(db1, db2)
'''
Check post-merge elements, make sure they match
'''
testmerge_check(db1, db2)
print "Finished test 1. "
def test_dataset():
#start with an empty collection
Dataset.drop_collection()
dataset = Dataset.make(language='en', name='/data/test')
# Make sure it was saved to the database
assert dataset.id is not None
# Make sure its attributes are readable
dataset.language
dataset.name
#test methods
reason = dataset.get_root_reason()
assert '/data/test/root' == reason
dataset.nl
dataset.check_consistency()
# Clean up by dropping the Dataset collection
Dataset.drop_collection()
def test_dataset():
#start with an empty collection
Dataset.drop_collection()
dataset = Dataset.make(language='en', name='/data/test')
# Make sure it was saved to the database
assert dataset.id is not None
# Make sure its attributes are readable
dataset.language
dataset.name
#test methods
reason = dataset.get_root_reason()
assert '/data/test/root' == reason
dataset.nl
dataset.check_consistency()
# Clean up by dropping the Dataset collection
Dataset.drop_collection()
def testmerge_make(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: Assertions 0-9
DB2: Assertions 0-4
'''
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
a.add_support(['/data/test/contributor/nholm'])
a0 = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i])
a0.add_support(['/data/test1/contributor/nholm'])
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(5):
a1 = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
a1.add_support(['/data/test/contributor/nholm'])
a2 = Assertion.make('/data/test','/rel/HasA',['/test/assertion','test/test%d'%i])
a2.add_support(['/data/test/contributor/nholm'])
a3 = Assertion.make('/data/test1','/rel/CausesDesire',['/test/assertion','test/test%d'%i])
a3.add_support(['/data/test1/contributor/nholm'])
def test_merge2(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: No assertions
DB2: No assertions
'''
print "Running test 2: "
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
#testmerge_display(db1, db2)
'''
Merge the two dbs
'''
merge(db1, db2)
'''
Check post-merge elements, make sure they match
'''
testmerge_check(db1, db2)
print "Finished test 2."
def test_assertion():
# fresh start
Dataset.drop_collection()
Assertion.drop_collection()
#create test dataset
dataset = Dataset.create(language='en', name='/data/test')
#make a test assertion
a1 = Assertion.make('/data/test', "/rel/IsA",
["/concept/test/assertion", "/concept/test/test"])
#verify that it exists in the database
assert a1.id is not None
#make sure attributes are readable
a1.dataset
a1.relation
a1.arguments
a1.argstr
a1.complete
a1.context
a1.polarity
a1.confidence
#make an identical test assertion
a2 = Assertion.make('/data/test', "/rel/IsA",
["/concept/test/assertion", "/concept/test/test"])
#verify that it exists in the database
assert a2.id is not None
print a2.id
print a1.id
print a1.argstr
#verify that attributes are readable
a2.dataset
a2.relation
a2.arguments
a2.argstr
a2.complete
a2.context
a2.polarity
a2.confidence
#check that all checked attributes are the same
assert a1.dataset == a2.dataset
assert a1.relation == a2.relation
assert a1.arguments == a2.arguments
assert a1.polarity == a2.polarity
assert a1.context == a2.context
#verify that the ID's are the same
assert (a1.id == a2.id)
#clean up
Dataset.drop_collection()
Assertion.drop_collection()
def test_assertion():
# fresh start
Dataset.drop_collection()
Assertion.drop_collection()
#create test dataset
dataset = Dataset.create(language = 'en', name = '/data/test')
#make a test assertion
a1 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])
#verify that it exists in the database
assert a1.id is not None
#make sure attributes are readable
a1.dataset
a1.relation
a1.arguments
a1.argstr
a1.complete
a1.context
a1.polarity
a1.confidence
#make an identical test assertion
a2 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])
#verify that it exists in the database
assert a2.id is not None
print a2.id
print a1.id
print a1.argstr
#verify that attributes are readable
a2.dataset
a2.relation
a2.arguments
a2.argstr
a2.complete
a2.context
a2.polarity
a2.confidence
#check that all checked attributes are the same
assert a1.dataset == a2.dataset
assert a1.relation == a2.relation
assert a1.arguments == a2.arguments
assert a1.polarity == a2.polarity
assert a1.context == a2.context
#verify that the ID's are the same
assert (a1.id == a2.id)
#clean up
Dataset.drop_collection()
Assertion.drop_collection()
def test_sentence():
dataset = Dataset.create(language='en', name='/data/test')
#create test sentence with dataset
sentence1 = Sentence.make('/data/test', "This is a test sentence.")
#check it was saved to the database
assert sentence1.id is not None
#make sure its attributes are readable
sentence1.text
sentence1.words
sentence1.dataset
sentence1.derived_assertions
sentence1.confidence
#make the same sentence, this time using dataset object instead of string
sentence2 = Sentence.make(dataset, "This is a test sentence.")
#check that it was saved to the database
assert sentence2.id is not None
#check that sentence1 and sentence2 have the same ID
assert (sentence1.id == sentence2.id)
#check that its attributes are readable
sentence2.text
sentence2.words
sentence2.dataset
sentence2.derived_assertions
sentence2.confidence
#make a different sentence
sentence3 = Sentence.make('/data/test', "This is a different test sentence.");
#make sure it exists in the database and is different
assert sentence3.id is not None
assert sentence3.id is not sentence1.id
assert sentence3.id is not sentence2.id
#make sure its attributes are readable
sentence3.text
sentence3.words
sentence3.dataset
sentence3.derived_assertions
sentence3.confidence
#clean up by dropping collections
Dataset.drop_collection()
Sentence.drop_collection()
def test_import_traversing():
Assertion.drop_collection()
Dataset.drop_collection()
q = MQLQuery.make({'mid': '/m/0p_47'}, ['*'])
# 'mid':'/m/0p_47'
q.get_results('/data/test', 'nholm', 1, None, True, 'mid')
#print 'DONE WITH GET RESULTS'
for a in Assertion.objects:
print a.relation
print a.arguments
Assertion.drop_collection()
Dataset.drop_collection()
def test_import_traversing():
Assertion.drop_collection()
Dataset.drop_collection()
q = MQLQuery.make({'mid':'/m/0p_47'},['*'])
# 'mid':'/m/0p_47'
q.get_results('/data/test', 'nholm', 1, None, True, 'mid')
#print 'DONE WITH GET RESULTS'
for a in Assertion.objects:
print a.relation
print a.arguments
Assertion.drop_collection()
Dataset.drop_collection()
def test_assertion():
# fresh start
Dataset.drop_collection()
Assertion.drop_collection()
#create test dataset
dataset = Dataset.create(language = 'en', name = '/data/test')
#make a test assertion
a1 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])
e = Expression.make(a1, "{0} is a {1}", ['this assertion', 'test'], 'en')
print e.assertion
e.add_support([dataset.get_root_reason()])
a1.add_support([dataset.get_root_reason()])
a1.save()
a1.make_generalizations('/data/test/root')
a2 = Assertion.objects.get(
dataset='/data/test',
relation='/rel/IsA',
argstr="/concept/test/assertion,/concept/test/test"
)
print a1.get_expressions()
assert a2.get_expressions()[0].text == 'this assertion is a test'
a3 = Assertion.objects.get(
dataset='/data/test',
relation='/rel/IsA',
argstr="/concept/test/assertion,*"
)
assert a3.get_expressions()[0].text == 'this assertion is a {1}'
a4 = Assertion.objects.get(
dataset='/data/test',
relation='/rel/IsA',
argstr="*,/concept/test/test"
)
assert a4.get_expressions()[0].text == '{0} is a test'
a5 = Assertion.objects.get(
dataset='/data/test',
relation='/rel/IsA',
argstr="*,*"
)
assert a5.get_expressions()[0].text == '{0} is a {1}'
#clean up
Dataset.drop_collection()
Assertion.drop_collection()
def import_activities(lang):
dataset = Dataset.make(DATASET_ROOT+lang, lang)
site = dataset.get_reason('/site/omcs')
for act in Activity.objects.iterator():
newact = dataset.get_reason(ACTIVITY_ROOT+act.name.replace(' ', '_'))
justify(newact, [site], 1.0)
log.info(newact)
def import_activities(lang):
dataset = Dataset.make(DATASET_ROOT + lang, lang)
site = dataset.get_reason('/site/omcs')
for act in Activity.objects.iterator():
newact = dataset.get_reason(ACTIVITY_ROOT + act.name.replace(' ', '_'))
justify(newact, [site], 1.0)
log.info(newact)
def datasetLookup(self,obj_url):
"""Method called when going to /api/data/{dataset name}. Returns
a serialized version of the dataset."""
try:
return Dataset.get(obj_url).serialize()
except DoesNotExist:
return rc.NOT_FOUND
def datasetLookup(self, obj_url):
"""Method called when going to /api/data/{dataset name}. Returns
a serialized version of the dataset."""
try:
return Dataset.get(obj_url).serialize()
except DoesNotExist:
return rc.NOT_FOUND
def import_assertions(lang, skip=0):
dataset = Dataset.make(DATASET_ROOT+lang, lang)
generalize_reason = dataset.get_reason('/rule/generalize')
justify(generalize_reason, [dataset.get_root_reason()], 0.1)
assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\
.select_related('concept1', 'concept2', 'relation', 'language')[skip:]
for assertion in assertions:
relation = RELATION_ROOT + assertion.relation.name
concept_names = [assertion.concept1.text, assertion.concept2.text]
concepts = [CONCEPT_ROOT+c for c in concept_names]
context = None
if -5 < assertion.frequency < 5:
context = '/concept/frequency/en/sometimes'
raws = assertion.rawassertion_set.all().select_related('surface1', 'surface2', 'frame', 'sentence', 'sentence__creator', 'sentence__activity')
newassertion = Assertion.make(dataset, relation, concepts,
polarity = assertion.polarity,
context=context)
newassertion.save()
sent_contributors = set()
support_votes = assertion.votes.filter(vote=1)
oppose_votes = assertion.votes.filter(vote=-1)
for vote in support_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
if voter not in sent_contributors:
newassertion.add_support([voter])
for vote in oppose_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
newassertion.add_oppose([voter])
for raw in raws:
if raw.score > 0:
frametext = raw.frame.text.replace('{1}','{0}').replace('{2}','{1}').replace('{%}','')
expr = newassertion.make_expression(frametext, [raw.surface1.text, raw.surface2.text], assertion.language.id)
support_votes = raw.votes.filter(vote=1).select_related('user')
oppose_votes = raw.votes.filter(vote=-1).select_related('user')
for vote in support_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
expr.add_support([voter])
for vote in oppose_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
expr.add_oppose([voter])
expr.save()
sent = raw.sentence
if sent.score > 0:
activity = sent.activity.name
act_reason = dataset.get_reason(ACTIVITY_ROOT+activity.replace(' ', '_'))
voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
sent_contributors.add(voter)
justification = [act_reason, voter]
newassertion.connect_to_sentence(dataset, sent.text, justification)
newassertion.make_generalizations(generalize_reason)
log.info(newassertion)
def test_merge6(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: Assertions 0-9
DB2: Assertions 0-4
'''
print "Running test 6: "
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
a.add_support(['/data/test/contributor/nholm'])
a0 = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i])
a0.add_support(['/data/test1/contributor/nholm'])
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(5):
a1 = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
a1.add_support(['/data/test/contributor/nholm'])
a2 = Assertion.make('/data/test','/rel/HasA',['/test/assertion','test/test%d'%i])
a2.add_support(['/data/test/contributor/nholm'])
a3 = Assertion.make('/data/test1','/rel/CausesDesire',['/test/assertion','test/test%d'%i])
a3.add_support(['/data/test1/contributor/nholm'])
#testmerge_display(db1, db2, '/data/test')
'''
Merge the two dbs
'''
merge(db1, db2, '/data/test')
'''
Check post-merge elements, make sure they match
'''
testmerge_check(db1, db2, '/data/test')
print "Finished test 6."
def import_contributors(lang):
dataset = Dataset.make(DATASET_ROOT+lang, lang)
root = dataset.get_root_reason()
site = dataset.get_reason('/site/omcs')
justify(site, [root], 1.0)
for user in User.objects.iterator():
contributor = dataset.get_reason(CONTRIBUTOR_ROOT+user.username)
justify(contributor, [site], 0.5)
log.info(contributor)
def import_contributors(lang):
dataset = Dataset.make(DATASET_ROOT + lang, lang)
root = dataset.get_root_reason()
site = dataset.get_reason('/site/omcs')
justify(site, [root], 1.0)
for user in User.objects.iterator():
contributor = dataset.get_reason(CONTRIBUTOR_ROOT + user.username)
justify(contributor, [site], 0.5)
log.info(contributor)
def fb_entity_property_from_id(self, dset, user, polarity=1, context=None):
'''
Called when the result field is not '*',
Only returns (property,value) pairs for properties
listed in the results_args and query_args,
Makes assertions for all of these pairs
'''
# create or get dataset
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [self.query_args]
assertionscreated = []
# start importing from freebase
mss = HTTPMetawebSession("http://api.freebase.com")
for searchterm in self.result_args:
query[0][searchterm] = {}
try:
results = mss.mqlread(query)
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % searchterm.capitalize(),
[self.query_args['id'], results[0][searchterm]['id']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
except MetawebError as me1:
if str(me1.args).rfind(
'/api/status/error/mql/result') is not -1:
query[0][searchterm] = [{}]
results = mss.mqlread(query)
for result in results[0][searchterm]:
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % searchterm.capitalize(),
[self.query_args['id'], result['id']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
elif str(me1.args).rfind(
'/api/status/error/mql/type') is not -1:
print 'The property %s is not recognized.' % searchterm
return
else:
print str(me1.args)
return
del query[0][searchterm]
return assertionscreated
def test_assertion():
# fresh start
Dataset.drop_collection()
Assertion.drop_collection()
#create test dataset
dataset = Dataset.create(language='en', name='/data/test')
#make a test assertion
a1 = Assertion.make('/data/test', "/rel/IsA",
["/concept/test/assertion", "/concept/test/test"])
e = Expression.make(a1, "{0} is a {1}", ['this assertion', 'test'], 'en')
print e.assertion
e.add_support([dataset.get_root_reason()])
a1.add_support([dataset.get_root_reason()])
a1.save()
a1.make_generalizations('/data/test/root')
a2 = Assertion.objects.get(
dataset='/data/test',
relation='/rel/IsA',
argstr="/concept/test/assertion,/concept/test/test")
print a1.get_expressions()
assert a2.get_expressions()[0].text == 'this assertion is a test'
a3 = Assertion.objects.get(dataset='/data/test',
relation='/rel/IsA',
argstr="/concept/test/assertion,*")
assert a3.get_expressions()[0].text == 'this assertion is a {1}'
a4 = Assertion.objects.get(dataset='/data/test',
relation='/rel/IsA',
argstr="*,/concept/test/test")
assert a4.get_expressions()[0].text == '{0} is a test'
a5 = Assertion.objects.get(dataset='/data/test',
relation='/rel/IsA',
argstr="*,*")
assert a5.get_expressions()[0].text == '{0} is a {1}'
#clean up
Dataset.drop_collection()
Assertion.drop_collection()
def import_sentence(sent):
dataset = Dataset.make(DATASET_ROOT+sent.language.id,
sent.language.id)
activity = sent.activity.name
root = dataset.get_root_reason()
site = root.derived_reason('/site/omcs')
act_reason = site.derived_reason(ACTIVITY_ROOT+activity.replace(' ', '_'))
contrib_reason = site.derived_reason(CONTRIBUTOR_ROOT+sent.creator.username)
justification = [act_reason, contrib_reason]
newsent = assertion.Sentence.make(dataset, sent.text, justification)
log.info(str(newsent))
def import_sentence(sent):
dataset = Dataset.make(DATASET_ROOT + sent.language.id, sent.language.id)
activity = sent.activity.name
root = dataset.get_root_reason()
site = root.derived_reason('/site/omcs')
act_reason = site.derived_reason(ACTIVITY_ROOT +
activity.replace(' ', '_'))
contrib_reason = site.derived_reason(CONTRIBUTOR_ROOT +
sent.creator.username)
justification = [act_reason, contrib_reason]
newsent = assertion.Sentence.make(dataset, sent.text, justification)
log.info(str(newsent))
def test_merge3(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: assertions in one dataset
DB2: assertions in another dataset
'''
print "Running test 3: "
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i])
a.add_support(['/data/test/contributor/nholm'])
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a1 = Assertion.make('/data/test2','/rel/IsA',['/test/assertion','test/test%d'%i])
a1.add_support(['/data/test/contributor/nholm'])
#testmerge_display(db1, db2)
'''
Merge the two dbs
'''
merge(db1, db2)
'''
Check post-merge elements, make sure they match
'''
testmerge_check(db1, db2)
print "Finished test 3."
def testmerge_make(db1='test1', db2='test2'):
'''
Load test assertions into the two DBs:
DB1: Assertions 0-9
DB2: Assertions 0-4
'''
conceptdb.create_mongodb(db1)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(10):
a = Assertion.make(
'/data/test', '/rel/IsA',
['/test/assertion', 'test/test%d' % i])
a.add_support(['/data/test/contributor/nholm'])
a0 = Assertion.make(
'/data/test1', '/rel/IsA',
['/test/assertion', 'test/test%d' % i])
a0.add_support(['/data/test1/contributor/nholm'])
conceptdb.create_mongodb(db2)
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
for i in xrange(5):
a1 = Assertion.make(
'/data/test', '/rel/IsA',
['/test/assertion', 'test/test%d' % i])
a1.add_support(['/data/test/contributor/nholm'])
a2 = Assertion.make(
'/data/test', '/rel/HasA',
['/test/assertion', 'test/test%d' % i])
a2.add_support(['/data/test/contributor/nholm'])
a3 = Assertion.make(
'/data/test1', '/rel/CausesDesire',
['/test/assertion', 'test/test%d' % i])
a3.add_support(['/data/test1/contributor/nholm'])
def fb_entity_property_from_id(self, dset, user, polarity=1, context=None):
'''
Called when the result field is not '*',
Only returns (property,value) pairs for properties
listed in the results_args and query_args,
Makes assertions for all of these pairs
'''
# create or get dataset
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [self.query_args]
assertionscreated = []
# start importing from freebase
mss = HTTPMetawebSession("http://api.freebase.com")
for searchterm in self.result_args:
query[0][searchterm]={}
try:
results = mss.mqlread(query)
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%searchterm.capitalize(), [self.query_args['id'],results[0][searchterm]['id']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
except MetawebError as me1:
if str(me1.args).rfind('/api/status/error/mql/result') is not -1:
query[0][searchterm]=[{}]
results = mss.mqlread(query)
for result in results[0][searchterm]:
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%searchterm.capitalize(), [self.query_args['id'],result['id']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
elif str(me1.args).rfind('/api/status/error/mql/type') is not -1:
print 'The property %s is not recognized.' % searchterm
return
else:
print str(me1.args)
return
del query[0][searchterm]
return assertionscreated
def make(dataset, text, reasons=None):
needs_save = False
if isinstance(dataset, basestring):
datasetObj = Dataset.get(dataset)
else:
datasetObj = dataset
dataset = datasetObj.name
try:
s = Sentence.objects.get(dataset=dataset, text=text)
except DoesNotExist:
s = Sentence(text=text,
dataset=dataset,
words=datasetObj.nl.normalize(text).split(),
derived_assertions=[])
needs_save = True
if reasons is not None:
s.add_support(reasons)
needs_save = True
if needs_save:
s.save()
return s
def make(dataset, text, reasons=None):
needs_save = False
if isinstance(dataset, basestring):
datasetObj = Dataset.get(dataset)
else:
datasetObj = dataset
dataset = datasetObj.name
try:
s = Sentence.objects.get(dataset=dataset, text=text)
except DoesNotExist:
s = Sentence(
text=text,
dataset=dataset,
words=datasetObj.nl.normalize(text).split(),
derived_assertions=[]
)
needs_save = True
if reasons is not None:
s.add_support(reasons)
needs_save = True
if needs_save:
s.save()
return s
import conceptdb
from conceptdb.assertion import Assertion, Expression, Sentence
from conceptdb.metadata import Dataset
from conceptdb.justify import ReasonConjunction
conceptdb.connect_to_mongodb('test')
#clean out whatever was in test before
Assertion.drop_collection()
Dataset.drop_collection()
Expression.drop_collection()
ReasonConjunction.drop_collection()
Sentence.drop_collection()
#make a new dataset
d = Dataset.make('/data/test', 'en')
#make a couple of assertions
a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test'])
a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test'])
#add expressions to them
e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language)
e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language)
#connect them to sentences
a1.connect_to_sentence(d, 'This assertion is a test.')
a2.connect_to_sentence(d, 'This database is a test.')
#make a reason for myself
r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'], 0.75, True)
def import_assertions(lang, skip=0):
dataset = Dataset.make(DATASET_ROOT + lang, lang)
generalize_reason = dataset.get_reason('/rule/generalize')
justify(generalize_reason, [dataset.get_root_reason()], 0.1)
assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\
.select_related('concept1', 'concept2', 'relation', 'language')[skip:]
for assertion in assertions:
relation = RELATION_ROOT + assertion.relation.name
concept_names = [assertion.concept1.text, assertion.concept2.text]
concepts = [CONCEPT_ROOT + c for c in concept_names]
context = None
if -5 < assertion.frequency < 5:
context = '/concept/frequency/en/sometimes'
raws = assertion.rawassertion_set.all().select_related(
'surface1', 'surface2', 'frame', 'sentence', 'sentence__creator',
'sentence__activity')
newassertion = Assertion.make(dataset,
relation,
concepts,
polarity=assertion.polarity,
context=context)
newassertion.save()
sent_contributors = set()
support_votes = assertion.votes.filter(vote=1)
oppose_votes = assertion.votes.filter(vote=-1)
for vote in support_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username)
if voter not in sent_contributors:
newassertion.add_support([voter])
for vote in oppose_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username)
newassertion.add_oppose([voter])
for raw in raws:
if raw.score > 0:
frametext = raw.frame.text.replace('{1}', '{0}').replace(
'{2}', '{1}').replace('{%}', '')
expr = newassertion.make_expression(
frametext, [raw.surface1.text, raw.surface2.text],
assertion.language.id)
support_votes = raw.votes.filter(vote=1).select_related('user')
oppose_votes = raw.votes.filter(vote=-1).select_related('user')
for vote in support_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT +
vote.user.username)
expr.add_support([voter])
for vote in oppose_votes:
voter = dataset.get_reason(CONTRIBUTOR_ROOT +
vote.user.username)
expr.add_oppose([voter])
expr.save()
sent = raw.sentence
if sent.score > 0:
activity = sent.activity.name
act_reason = dataset.get_reason(ACTIVITY_ROOT +
activity.replace(' ', '_'))
voter = dataset.get_reason(CONTRIBUTOR_ROOT +
vote.user.username)
sent_contributors.add(voter)
justification = [act_reason, voter]
newassertion.connect_to_sentence(dataset, sent.text,
justification)
newassertion.make_generalizations(generalize_reason)
log.info(newassertion)
def fb_entity_from_mid(self, dset, user, polarity=1, context=None):
'''
Called when the result field is '*'
Returns all of the results (property-value) pairs for the given mid object,
Makes assertions of these results
'''
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [dict(self.query_args,**{'*':{}})]
assertionscreated = []
# start importing from freebase
mss = freebase.HTTPMetawebSession("http://api.freebase.com")
try:
results = mss.mqlread(query)
except MetawebError:
print 'MetawebError occurred at %s'%str(query)
return []
#print results[0]
if type(results)==list:
results = results[0]
#for key in self.query_args.keys():
# initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['mid'],self.query_args[key]])
# initial_assertion.add_support(dataset.name + '/contributor/' + user)
# assertionscreated.append(initial_assertion)
for property in [r for r in results if r not in self.skip_props and r not in self.query_args.keys()]:
#print property
if type(results[property])==list:
for value in results[property]:
try:
#print self.query_args['id']
#print value['id']
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],value['id']])
except:
#print self.query_args['id']
#print value['id']
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],value['value']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
else:
#print results[property]
try:
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],results[property]['value']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
#print 'second exception'
#a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
try:
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],results[property]['id']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
pass
return assertionscreated
def testmerge_check(db1='test1', db2='test2', dataset=None):
'''
Check post-merge elements, make sure they match
'''
if dataset==None:
print "Testing Assertion objects"
conceptdb.connect_to_mongodb(db1)
db1_assertions = Assertion.objects
conceptdb.connect_to_mongodb(db2)
db2_assertions = Assertion.objects
for db1_a in [a1 for a1 in list(db1_assertions) if a1 not in list(db2_assertions)]:
check = False
for db2_check in list(db2_assertions):
if assertion_check(db1_a, db2_check):
check = True
assert check==True
for db2_a in [a2 for a2 in list(db2_assertions) if a2 not in list(db1_assertions)]:
check = False
for db1_check in list(db1_assertions):
if assertion_check(db2_a, db1_check):
check = True
assert check==True
print "Assertion test PASSED"
print "Testing ReasonConjunction objects"
conceptdb.connect_to_mongodb(db1)
#print "After the merge, db %s has the following assertions: "%db1
for a1 in Assertion.objects:
#print "assertion: %s"%a1
#print " confidence score: %s"%a1.confidence
for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
#print " reason: %s"%r1.factors
assert r1.target == a1.name
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.connect_to_mongodb(db2)
#print "After the merge, db %s has the following assertions: "%db2
for a2 in Assertion.objects:
#print "assertion: %s"%a2
#print " confidence score: %s"%a2.confidence
for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
#print " reason: %s"%r2.factors
assert r2.target == a2.name
print "ReasonConjunction test PASSED"
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
else:
print "Testing Assertion objects"
conceptdb.connect_to_mongodb(db1)
db1_assertions = Assertion.objects.filter(dataset=dataset)
conceptdb.connect_to_mongodb(db2)
db2_assertions = Assertion.objects.filter(dataset=dataset)
for db1_a in [a1 for a1 in list(db1_assertions) if a1 not in list(db2_assertions)]:
check = False
for db2_check in list(db2_assertions):
if assertion_check(db1_a, db2_check):
check = True
assert check==True
for db2_a in [a2 for a2 in list(db2_assertions) if a2 not in list(db1_assertions)]:
check = False
for db1_check in list(db1_assertions):
if assertion_check(db2_a, db1_check):
check = True
assert check==True
print "Assertion test PASSED"
print "Testing ReasonConjunction objects"
conceptdb.connect_to_mongodb(db1)
#print "After the merge, db %s has the following assertions: "%db1
for a1 in Assertion.objects.filter(dataset=dataset):
#print "assertion: %s"%a1
#print " confidence score: %s"%a1.confidence
for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
#print " reason: %s"%r1.factors
assert r1.target == a1.name
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.connect_to_mongodb(db2)
#print "After the merge, db %s has the following assertions: "%db2
for a2 in Assertion.objects.filter(dataset=dataset):
#print "assertion: %s"%a2
#print " confidence score: %s"%a2.confidence
for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
#print " reason: %s"%r2.factors
assert r2.target == a2.name
print "ReasonConjunction test PASSED"
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
import conceptdb
from conceptdb.assertion import Assertion, Expression, Sentence
from conceptdb.metadata import Dataset
from conceptdb.justify import ReasonConjunction
conceptdb.connect_to_mongodb('test')
#clean out whatever was in test before
Assertion.drop_collection()
Dataset.drop_collection()
Expression.drop_collection()
ReasonConjunction.drop_collection()
Sentence.drop_collection()
#make a new dataset
d = Dataset.make('/data/test', 'en')
#make a couple of assertions
a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test'])
a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test'])
#add expressions to them
e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language)
e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language)
#connect them to sentences
a1.connect_to_sentence(d, 'This assertion is a test.')
a2.connect_to_sentence(d, 'This database is a test.')
#make a reason for myself
r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'],
def fb_entity_from_mid(self, dset, user, polarity=1, context=None):
'''
Called when the result field is '*'
Returns all of the results (property-value) pairs for the given mid object,
Makes assertions of these results
'''
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [dict(self.query_args, **{'*': {}})]
assertionscreated = []
# start importing from freebase
mss = freebase.HTTPMetawebSession("http://api.freebase.com")
try:
results = mss.mqlread(query)
except MetawebError:
print 'MetawebError occurred at %s' % str(query)
return []
#print results[0]
if type(results) == list:
results = results[0]
#for key in self.query_args.keys():
# initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['mid'],self.query_args[key]])
# initial_assertion.add_support(dataset.name + '/contributor/' + user)
# assertionscreated.append(initial_assertion)
for property in [
r for r in results if r not in self.skip_props
and r not in self.query_args.keys()
]:
#print property
if type(results[property]) == list:
for value in results[property]:
try:
#print self.query_args['id']
#print value['id']
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['mid'], value['id']])
except:
#print self.query_args['id']
#print value['id']
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['mid'], value['value']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
else:
#print results[property]
try:
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['mid'], results[property]['value']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
#print 'second exception'
#a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
try:
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['mid'], results[property]['id']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
pass
return assertionscreated
def fb_entity_from_id(self, dset, user, polarity=1, context=None):
'''
Called when the result field is '*'
Returns all of the results (property-value) pairs for the given id object,
Makes assertions of these results
'''
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [dict(self.query_args, **{'*': {}})]
assertionscreated = []
# start importing from freebase
mss = freebase.HTTPMetawebSession("http://api.freebase.com")
results = mss.mqlread(query)
#print 'RESULTS ARE'
#print results
if type(results) == list:
results = results[0]
for key in self.query_args.keys():
initial_assertion = Assertion.make(
dataset.name, '/rel/freebase/has%s' % key.capitalize(),
[self.query_args['id'], self.query_args[key]])
initial_assertion.add_support(dataset.name + '/contributor/' +
user)
assertionscreated.append(initial_assertion)
#print 'MADE INITIAL ASSERTION'
# Go through all properties, excluding properties in the skip_props list and properties whose results are not of type list
for property in [
r for r in results if r not in self.skip_props
and r not in self.query_args.keys()
]:
# Use properties to index concepts, and only use concepts with an explicit 'mmid' field
if type(results[property]) == list:
for value in results[property]:
try:
#print self.query_args['id']
#print value['id']
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['id'], value['id']])
except:
#print self.query_args['id']
#print value['id']
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['id'], value['value']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
else:
#print results[property]
try:
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['id'], results[property]['value']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
#print 'second exception'
#a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
try:
a = Assertion.make(
dataset.name,
'/rel/freebase/has%s' % property.capitalize(),
[self.query_args['id'], results[property]['id']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
pass
return assertionscreated
def fb_entity_from_id(self, dset, user, polarity=1, context=None):
'''
Called when the result field is '*'
Returns all of the results (property-value) pairs for the given id object,
Makes assertions of these results
'''
try:
dataset = Dataset.objects.get(name=dset, language='en')
except DoesNotExist:
dataset = Dataset.create(name=dset, language='en')
query = [dict(self.query_args,**{'*':{}})]
assertionscreated = []
# start importing from freebase
mss = freebase.HTTPMetawebSession("http://api.freebase.com")
results = mss.mqlread(query)
#print 'RESULTS ARE'
#print results
if type(results)==list:
results = results[0]
for key in self.query_args.keys():
initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['id'],self.query_args[key]])
initial_assertion.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(initial_assertion)
#print 'MADE INITIAL ASSERTION'
# Go through all properties, excluding properties in the skip_props list and properties whose results are not of type list
for property in [r for r in results if r not in self.skip_props and r not in self.query_args.keys()]:
# Use properties to index concepts, and only use concepts with an explicit 'mmid' field
if type(results[property])==list:
for value in results[property]:
try:
#print self.query_args['id']
#print value['id']
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],value['id']])
except:
#print self.query_args['id']
#print value['id']
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],value['value']])
a.add_support(dataset.name + '/contributor/' + user)
assertionscreated.append(a)
else:
#print results[property]
try:
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],results[property]['value']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
#print 'second exception'
#a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
try:
a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],results[property]['id']])
a.add_support(dataset.name + '/contributor/' + 'nholm')
assertionscreated.append(a)
except:
pass
return assertionscreated
def testmerge_check(db1='test1', db2='test2', dataset=None):
'''
Check post-merge elements, make sure they match
'''
if dataset == None:
print "Testing Assertion objects"
conceptdb.connect_to_mongodb(db1)
db1_assertions = Assertion.objects
conceptdb.connect_to_mongodb(db2)
db2_assertions = Assertion.objects
for db1_a in [
a1 for a1 in list(db1_assertions)
if a1 not in list(db2_assertions)
]:
check = False
for db2_check in list(db2_assertions):
if assertion_check(db1_a, db2_check):
check = True
assert check == True
for db2_a in [
a2 for a2 in list(db2_assertions)
if a2 not in list(db1_assertions)
]:
check = False
for db1_check in list(db1_assertions):
if assertion_check(db2_a, db1_check):
check = True
assert check == True
print "Assertion test PASSED"
print "Testing ReasonConjunction objects"
conceptdb.connect_to_mongodb(db1)
#print "After the merge, db %s has the following assertions: "%db1
for a1 in Assertion.objects:
#print "assertion: %s"%a1
#print " confidence score: %s"%a1.confidence
for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
#print " reason: %s"%r1.factors
assert r1.target == a1.name
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.connect_to_mongodb(db2)
#print "After the merge, db %s has the following assertions: "%db2
for a2 in Assertion.objects:
#print "assertion: %s"%a2
#print " confidence score: %s"%a2.confidence
for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
#print " reason: %s"%r2.factors
assert r2.target == a2.name
print "ReasonConjunction test PASSED"
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
else:
print "Testing Assertion objects"
conceptdb.connect_to_mongodb(db1)
db1_assertions = Assertion.objects.filter(dataset=dataset)
conceptdb.connect_to_mongodb(db2)
db2_assertions = Assertion.objects.filter(dataset=dataset)
for db1_a in [
a1 for a1 in list(db1_assertions)
if a1 not in list(db2_assertions)
]:
check = False
for db2_check in list(db2_assertions):
if assertion_check(db1_a, db2_check):
check = True
assert check == True
for db2_a in [
a2 for a2 in list(db2_assertions)
if a2 not in list(db1_assertions)
]:
check = False
for db1_check in list(db1_assertions):
if assertion_check(db2_a, db1_check):
check = True
assert check == True
print "Assertion test PASSED"
print "Testing ReasonConjunction objects"
conceptdb.connect_to_mongodb(db1)
#print "After the merge, db %s has the following assertions: "%db1
for a1 in Assertion.objects.filter(dataset=dataset):
#print "assertion: %s"%a1
#print " confidence score: %s"%a1.confidence
for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
#print " reason: %s"%r1.factors
assert r1.target == a1.name
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()
conceptdb.connect_to_mongodb(db2)
#print "After the merge, db %s has the following assertions: "%db2
for a2 in Assertion.objects.filter(dataset=dataset):
#print "assertion: %s"%a2
#print " confidence score: %s"%a2.confidence
for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
#print " reason: %s"%r2.factors
assert r2.target == a2.name
print "ReasonConjunction test PASSED"
Assertion.drop_collection()
Dataset.drop_collection()
ReasonConjunction.drop_collection()