Example #1
0
def test_merge2(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: No assertions
    DB2: No assertions
    '''
    print "Running test 2: "
    
    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()  
    ReasonConjunction.drop_collection()

    
    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()

    
    #testmerge_display(db1, db2)
    
    '''
    Merge the two dbs
    '''
    merge(db1, db2)
    
    '''
    Check post-merge elements, make sure they match
    '''
    
    testmerge_check(db1, db2)
    
    print "Finished test 2."
Example #2
0
def test_merge2(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: No assertions
    DB2: No assertions
    '''
    print "Running test 2: "

    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()

    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()

    #testmerge_display(db1, db2)
    '''
    Merge the two dbs
    '''
    merge(db1, db2)
    '''
    Check post-merge elements, make sure they match
    '''

    testmerge_check(db1, db2)

    print "Finished test 2."
Example #3
0
def test_merge1(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: No assertions
    DB2: Assertions 0-9
    '''
    print "Running test 1: "

    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()

    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a1 = Assertion.make(
            '/data/test', '/rel/IsA',
            ['/test/assertion', 'test/test%d' % i])
        a1.add_support(['/data/test/contributor/nholm'])

    #testmerge_display(db1, db2)
    '''
    Merge the two dbs
    '''
    merge(db1, db2)
    '''
    Check post-merge elements, make sure they match
    '''

    testmerge_check(db1, db2)

    print "Finished test 1. "
Example #4
0
    def fb_entity_property_from_id(self, dset, user, polarity=1, context=None):
        '''
        Called when the result field is not '*',
        Only returns (property,value) pairs for properties 
        listed in the results_args and query_args,
        Makes assertions for all of these pairs
        '''
        # create or get dataset
        try:
            dataset = Dataset.objects.get(name=dset, language='en')
        except DoesNotExist:
            dataset = Dataset.create(name=dset, language='en')

        query = [self.query_args]

        assertionscreated = []

        # start importing from freebase
        mss = HTTPMetawebSession("http://api.freebase.com")

        for searchterm in self.result_args:
            query[0][searchterm] = {}
            try:
                results = mss.mqlread(query)
                a = Assertion.make(
                    dataset.name,
                    '/rel/freebase/has%s' % searchterm.capitalize(),
                    [self.query_args['id'], results[0][searchterm]['id']])
                a.add_support(dataset.name + '/contributor/' + user)
                assertionscreated.append(a)

            except MetawebError as me1:
                if str(me1.args).rfind(
                        '/api/status/error/mql/result') is not -1:
                    query[0][searchterm] = [{}]
                    results = mss.mqlread(query)
                    for result in results[0][searchterm]:
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % searchterm.capitalize(),
                            [self.query_args['id'], result['id']])
                        a.add_support(dataset.name + '/contributor/' + user)
                        assertionscreated.append(a)

                elif str(me1.args).rfind(
                        '/api/status/error/mql/type') is not -1:
                    print 'The property %s is not recognized.' % searchterm
                    return

                else:
                    print str(me1.args)
                    return

            del query[0][searchterm]
        return assertionscreated
Example #5
0
def testmerge_make(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: Assertions 0-9
    DB2: Assertions 0-4
    '''
    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()  
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
        a.add_support(['/data/test/contributor/nholm'])    
        a0 = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i]) 
        a0.add_support(['/data/test1/contributor/nholm'])    

    
    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(5):
        a1 = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])  
        a1.add_support(['/data/test/contributor/nholm'])
        a2 = Assertion.make('/data/test','/rel/HasA',['/test/assertion','test/test%d'%i])  
        a2.add_support(['/data/test/contributor/nholm'])  
        a3 = Assertion.make('/data/test1','/rel/CausesDesire',['/test/assertion','test/test%d'%i])  
        a3.add_support(['/data/test1/contributor/nholm'])
Example #6
0
def test_import_traversing():
    Assertion.drop_collection()
    Dataset.drop_collection()

    q = MQLQuery.make({'mid': '/m/0p_47'}, ['*'])
    # 'mid':'/m/0p_47'
    q.get_results('/data/test', 'nholm', 1, None, True, 'mid')

    #print 'DONE WITH GET RESULTS'
    for a in Assertion.objects:
        print a.relation
        print a.arguments

    Assertion.drop_collection()
    Dataset.drop_collection()
Example #7
0
def test_freebase_allresults():
    Assertion.drop_collection()

    query_args = {'id': '/en/the_beatles', 'type': '/music/artist'}
    result_args = ['*']

    q = MQLQuery.make(query_args, result_args)

    q.get_results('/data/test')

    for a in Assertion.objects:
        print str(a.arguments)
        print str(a.relation)

    Assertion.drop_collection()
Example #8
0
def test_import_traversing():
    Assertion.drop_collection()
    Dataset.drop_collection()
    
    q = MQLQuery.make({'mid':'/m/0p_47'},['*'])
    # 'mid':'/m/0p_47'
    q.get_results('/data/test', 'nholm', 1, None, True, 'mid')
    
    #print 'DONE WITH GET RESULTS'
    for a in Assertion.objects:
        print a.relation
        print a.arguments

    Assertion.drop_collection()
    Dataset.drop_collection()
Example #9
0
def test_freebase_allresults():
    Assertion.drop_collection()
    
    query_args = {'id':'/en/the_beatles', 'type':'/music/artist'}
    result_args = ['*']
    
    q = MQLQuery.make(query_args, result_args)
    
    q.get_results('/data/test')
    
    for a in Assertion.objects:
        print str(a.arguments)
        print str(a.relation)
    
    Assertion.drop_collection()
Example #10
0
def test_assertion():
    # fresh start
    Dataset.drop_collection()
    Assertion.drop_collection()
    
    #create test dataset
    dataset = Dataset.create(language = 'en', name = '/data/test')

    #make a test assertion
    a1 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])
    e = Expression.make(a1, "{0} is a {1}", ['this assertion', 'test'], 'en')
    print e.assertion
    e.add_support([dataset.get_root_reason()])
    a1.add_support([dataset.get_root_reason()])
    a1.save()
    a1.make_generalizations('/data/test/root')
    a2 = Assertion.objects.get(
        dataset='/data/test',
        relation='/rel/IsA',
        argstr="/concept/test/assertion,/concept/test/test"
    )
    print a1.get_expressions()
    assert a2.get_expressions()[0].text == 'this assertion is a test'

    a3 = Assertion.objects.get(
        dataset='/data/test',
        relation='/rel/IsA',
        argstr="/concept/test/assertion,*"
    )
    assert a3.get_expressions()[0].text == 'this assertion is a {1}'
    
    a4 = Assertion.objects.get(
        dataset='/data/test',
        relation='/rel/IsA',
        argstr="*,/concept/test/test"
    )
    assert a4.get_expressions()[0].text == '{0} is a test'
    
    a5 = Assertion.objects.get(
        dataset='/data/test',
        relation='/rel/IsA',
        argstr="*,*"
    )
    assert a5.get_expressions()[0].text == '{0} is a {1}'
    
    #clean up
    Dataset.drop_collection()
    Assertion.drop_collection()
Example #11
0
 def fb_entity_property_from_id(self, dset, user, polarity=1, context=None):
     '''
     Called when the result field is not '*',
     Only returns (property,value) pairs for properties 
     listed in the results_args and query_args,
     Makes assertions for all of these pairs
     '''
     # create or get dataset
     try:
         dataset = Dataset.objects.get(name=dset, language='en')
     except DoesNotExist:
         dataset = Dataset.create(name=dset, language='en')
     
     query = [self.query_args]
     
     assertionscreated = []
     
     # start importing from freebase
     mss = HTTPMetawebSession("http://api.freebase.com")
     
     for searchterm in self.result_args:
         query[0][searchterm]={}
         try:    
             results = mss.mqlread(query)
             a = Assertion.make(dataset.name, '/rel/freebase/has%s'%searchterm.capitalize(), [self.query_args['id'],results[0][searchterm]['id']])
             a.add_support(dataset.name + '/contributor/' + user)           
             assertionscreated.append(a)
     
         except MetawebError as me1:
             if str(me1.args).rfind('/api/status/error/mql/result') is not -1:
                 query[0][searchterm]=[{}]
                 results = mss.mqlread(query)
                 for result in results[0][searchterm]:
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%searchterm.capitalize(), [self.query_args['id'],result['id']])
                     a.add_support(dataset.name + '/contributor/' + user)
                     assertionscreated.append(a)
             
             elif str(me1.args).rfind('/api/status/error/mql/type') is not -1:
                 print 'The property %s is not recognized.' % searchterm
                 return
         
             else:
                 print str(me1.args)
                 return
     
         del query[0][searchterm]
     return assertionscreated
Example #12
0
def import_assertions(lang, skip=0):
    dataset = Dataset.make(DATASET_ROOT+lang, lang)
    generalize_reason = dataset.get_reason('/rule/generalize')
    justify(generalize_reason, [dataset.get_root_reason()], 0.1)

    assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\
      .select_related('concept1', 'concept2', 'relation', 'language')[skip:]
    for assertion in assertions:
        relation = RELATION_ROOT + assertion.relation.name
        concept_names = [assertion.concept1.text, assertion.concept2.text]
        concepts = [CONCEPT_ROOT+c for c in concept_names]
        context = None
        if -5 < assertion.frequency < 5:
            context = '/concept/frequency/en/sometimes'
        raws = assertion.rawassertion_set.all().select_related('surface1', 'surface2', 'frame', 'sentence', 'sentence__creator', 'sentence__activity')

        newassertion = Assertion.make(dataset, relation, concepts,
                                         polarity = assertion.polarity,
                                         context=context)
        newassertion.save()
        
        sent_contributors = set()
        support_votes = assertion.votes.filter(vote=1)
        oppose_votes = assertion.votes.filter(vote=-1)
        for vote in support_votes:
            voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
            if voter not in sent_contributors:
                newassertion.add_support([voter])
        for vote in oppose_votes:
            voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
            newassertion.add_oppose([voter])

        for raw in raws:
            if raw.score > 0:
                frametext = raw.frame.text.replace('{1}','{0}').replace('{2}','{1}').replace('{%}','')
                expr = newassertion.make_expression(frametext, [raw.surface1.text, raw.surface2.text], assertion.language.id)
                support_votes = raw.votes.filter(vote=1).select_related('user')
                oppose_votes = raw.votes.filter(vote=-1).select_related('user')
                for vote in support_votes:
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
                    expr.add_support([voter])
                for vote in oppose_votes:
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)
                    expr.add_oppose([voter])
                expr.save()

                sent = raw.sentence
                if sent.score > 0:
                    activity = sent.activity.name
                    act_reason = dataset.get_reason(ACTIVITY_ROOT+activity.replace(' ', '_'))
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username)

                    sent_contributors.add(voter)
                    justification = [act_reason, voter]
                    newassertion.connect_to_sentence(dataset, sent.text, justification)

        newassertion.make_generalizations(generalize_reason)
        log.info(newassertion)
Example #13
0
    def assertionMake(self, request, obj_url):
        """This method takes the unique identifiers of an assertion as its arguments:
        dataset, relation, concepts, context, and polarity.  It checks to see if this
        assertion exists.  If it does not, it creates it and adds the submitting user's
        vote as a justification.  If it exists already, it adds the submitting user's
        vote as a justification.

        Accessed by going to the URL
        /api/assertionmake?dataset={dataset}&rel={relation}&concepts={concept1,concept2,etc}&
        polarity={polarity}&context={context}&user={username}&password={password}

        Polarity and context are optional, defaulting to polarity = 1 context = None
        """
        dataset = request.POST['dataset']
        relation = request.POST['rel']
        argstr = request.POST['concepts']
        arguments = argstr.split(',')
        polarity = int(request.POST.get('polarity', '1'))
        context = request.POST.get('context', 'None')
        user = request.POST['user']
        password = request.POST['password']

        if context == "None":
            context = None
        # TODO: uncomment to take into account user and password?
        if User.objects.get(username=user).check_password(password):
            #the user's password is correct.  Get their reason and add

            try:
                user_reason = ReasonConjunction.objects.get(
                    target=dataset + '/contributor/' + user)
            except DoesNotExist:
                return rc.FORBIDDEN
        else:
            #incorrect password
            return rc.FORBIDDEN

        try:
            assertion = Assertion.objects.get(dataset=dataset,
                                              relation=relation,
                                              argstr=argstr,
                                              polarity=polarity,
                                              context=context)

            assertion.add_support([dataset + '/contributor/' + user])
            return "The assertion you created already exists.  Your vote for this \
            assertion has been counted.\n" + str(assertion.serialize())

        except DoesNotExist:
            assertion = Assertion.make(dataset=dataset,
                                       arguments=arguments,
                                       relation=relation,
                                       polarity=polarity,
                                       context=context)

            assertion.add_support([dataset + '/contributor/' + user])

            return assertion.serialize()
Example #14
0
    def assertionMake(self, request, obj_url):
        """This method takes the unique identifiers of an assertion as its arguments:
        dataset, relation, concepts, context, and polarity.  It checks to see if this
        assertion exists.  If it does not, it creates it and adds the submitting user's
        vote as a justification.  If it exists already, it adds the submitting user's
        vote as a justification.

        Accessed by going to the URL
        /api/assertionmake?dataset={dataset}&rel={relation}&concepts={concept1,concept2,etc}&
        polarity={polarity}&context={context}&user={username}&password={password}

        Polarity and context are optional, defaulting to polarity = 1 context = None
        """
        dataset = request.POST['dataset']
        relation = request.POST['rel']
        argstr = request.POST['concepts']
        arguments = argstr.split(',')
        polarity = int(request.POST.get('polarity','1'))
        context = request.POST.get('context','None')
        user = request.POST['user']
        password = request.POST['password']

        if context == "None":
            context = None
        # TODO: uncomment to take into account user and password?
        if User.objects.get(username=user).check_password(password):
            #the user's password is correct.  Get their reason and add
            
            try:
                user_reason = ReasonConjunction.objects.get(target=dataset + '/contributor/' + user)
            except DoesNotExist:
                return rc.FORBIDDEN
        else:
            #incorrect password
            return rc.FORBIDDEN
        
        try:
            assertion = Assertion.objects.get(
                dataset = dataset,
                relation = relation,
                argstr = argstr,
                polarity = polarity,
                context = context)
            
            assertion.add_support([dataset + '/contributor/' + user]) 
            return "The assertion you created already exists.  Your vote for this \
            assertion has been counted.\n" + str(assertion.serialize())

        except DoesNotExist:
            assertion = Assertion.make(dataset = dataset,
                        arguments = arguments,
                        relation = relation,
                        polarity = polarity,
                        context = context)
            
            assertion.add_support([dataset + '/contributor/' + user]) 

            return assertion.serialize()
Example #15
0
 def assertionLookup(self, obj_url):
     """Method called to look up an assertion by its id number.  Accessed
     by going to URL /api/assertion/{id}.  Returns a serialized version 
     of the assertion."""
     try:
         return Assertion.get(obj_url.replace('/assertion/', '')).serialize()    
     except DoesNotExist:
         return rc.NOT_FOUND
     except ValidationError: #raised if input is not a valid id
         return rc.NOT_FOUND
Example #16
0
def test_create_or_vote():
    q = MQLQuery.make({'id': '/en/the_beatles'}, ['*'])

    Assertion.drop_collection()

    assertions = q.get_results('/data/test', 'nholm', 1, None, False)

    print str(len(assertions))

    assertions2 = q.get_results('/data/test', 'nholm', 1, None, False)

    print str(len(assertions2))
    count = 0
    for a in Assertion.objects:
        count += 1
        print a.arguments
    print count

    Assertion.drop_collection()
Example #17
0
def test_create_or_vote():
    q = MQLQuery.make({'id':'/en/the_beatles'}, ['*'])
    
    Assertion.drop_collection()
    
    assertions = q.get_results('/data/test','nholm', 1,None,False)
    
    print str(len(assertions))
    
    assertions2 = q.get_results('/data/test','nholm', 1,None,False)
    
    print str(len(assertions2))
    count = 0
    for a in Assertion.objects:
        count += 1
        print a.arguments
    print count
    
    Assertion.drop_collection()
Example #18
0
def test_assertion():
    # fresh start
    Dataset.drop_collection()
    Assertion.drop_collection()

    #create test dataset
    dataset = Dataset.create(language='en', name='/data/test')

    #make a test assertion
    a1 = Assertion.make('/data/test', "/rel/IsA",
                        ["/concept/test/assertion", "/concept/test/test"])
    e = Expression.make(a1, "{0} is a {1}", ['this assertion', 'test'], 'en')
    print e.assertion
    e.add_support([dataset.get_root_reason()])
    a1.add_support([dataset.get_root_reason()])
    a1.save()
    a1.make_generalizations('/data/test/root')
    a2 = Assertion.objects.get(
        dataset='/data/test',
        relation='/rel/IsA',
        argstr="/concept/test/assertion,/concept/test/test")
    print a1.get_expressions()
    assert a2.get_expressions()[0].text == 'this assertion is a test'

    a3 = Assertion.objects.get(dataset='/data/test',
                               relation='/rel/IsA',
                               argstr="/concept/test/assertion,*")
    assert a3.get_expressions()[0].text == 'this assertion is a {1}'

    a4 = Assertion.objects.get(dataset='/data/test',
                               relation='/rel/IsA',
                               argstr="*,/concept/test/test")
    assert a4.get_expressions()[0].text == '{0} is a test'

    a5 = Assertion.objects.get(dataset='/data/test',
                               relation='/rel/IsA',
                               argstr="*,*")
    assert a5.get_expressions()[0].text == '{0} is a {1}'

    #clean up
    Dataset.drop_collection()
    Assertion.drop_collection()
Example #19
0
 def assertionLookup(self, obj_url):
     """Method called to look up an assertion by its id number.  Accessed
     by going to URL /api/assertion/{id}.  Returns a serialized version 
     of the assertion."""
     try:
         return Assertion.get(obj_url.replace('/assertion/',
                                              '')).serialize()
     except DoesNotExist:
         return rc.NOT_FOUND
     except ValidationError:  #raised if input is not a valid id
         return rc.NOT_FOUND
Example #20
0
def test_import_all():
    
    Assertion.drop_collection()
    
    q = MQLQuery.make({'id':'/en/the_beatles'}, ['*'])
    
    assertions = q.get_results('/data/test',1,None,'nholm',True)
    
    for a in Assertion.objects:
        print a.relation
#
#    mss = freebase.HTTPMetawebSession('http://api.freebase.com')
#    
#    query = [{"*":{},"id":"/en/the_beatles","type":"/music/artist"}]
#    
#    results = mss.mqlread(query)
#    
#    print results

    
    Assertion.drop_collection()
Example #21
0
def test_merge6(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: Assertions 0-9
    DB2: Assertions 0-4
    '''
    print "Running test 6: "
    
    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()  
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])
        a.add_support(['/data/test/contributor/nholm'])    
        a0 = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i]) 
        a0.add_support(['/data/test1/contributor/nholm'])    

    
    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(5):
        a1 = Assertion.make('/data/test','/rel/IsA',['/test/assertion','test/test%d'%i])  
        a1.add_support(['/data/test/contributor/nholm'])
        a2 = Assertion.make('/data/test','/rel/HasA',['/test/assertion','test/test%d'%i])  
        a2.add_support(['/data/test/contributor/nholm'])  
        a3 = Assertion.make('/data/test1','/rel/CausesDesire',['/test/assertion','test/test%d'%i])  
        a3.add_support(['/data/test1/contributor/nholm'])
        
        
    #testmerge_display(db1, db2, '/data/test')
    
    '''
    Merge the two dbs
    '''
    merge(db1, db2, '/data/test')
    
    '''
    Check post-merge elements, make sure they match
    '''
    testmerge_check(db1, db2, '/data/test')
    
    print "Finished test 6."
Example #22
0
def test_import_all():

    Assertion.drop_collection()

    q = MQLQuery.make({'id': '/en/the_beatles'}, ['*'])

    assertions = q.get_results('/data/test', 1, None, 'nholm', True)

    for a in Assertion.objects:
        print a.relation


#
#    mss = freebase.HTTPMetawebSession('http://api.freebase.com')
#
#    query = [{"*":{},"id":"/en/the_beatles","type":"/music/artist"}]
#
#    results = mss.mqlread(query)
#
#    print results

    Assertion.drop_collection()
Example #23
0
def test_assertion():
    # fresh start
    Dataset.drop_collection()
    Assertion.drop_collection()

    #create test dataset
    dataset = Dataset.create(language='en', name='/data/test')

    #make a test assertion
    a1 = Assertion.make('/data/test', "/rel/IsA",
                        ["/concept/test/assertion", "/concept/test/test"])

    #verify that it exists in the database
    assert a1.id is not None

    #make sure attributes are readable
    a1.dataset
    a1.relation
    a1.arguments
    a1.argstr
    a1.complete
    a1.context
    a1.polarity
    a1.confidence

    #make an identical test assertion
    a2 = Assertion.make('/data/test', "/rel/IsA",
                        ["/concept/test/assertion", "/concept/test/test"])

    #verify that it exists in the database
    assert a2.id is not None
    print a2.id
    print a1.id
    print a1.argstr

    #verify that attributes are readable
    a2.dataset
    a2.relation
    a2.arguments
    a2.argstr
    a2.complete
    a2.context
    a2.polarity
    a2.confidence

    #check that all checked attributes are the same
    assert a1.dataset == a2.dataset
    assert a1.relation == a2.relation
    assert a1.arguments == a2.arguments
    assert a1.polarity == a2.polarity
    assert a1.context == a2.context

    #verify that the ID's are the same
    assert (a1.id == a2.id)

    #clean up
    Dataset.drop_collection()
    Assertion.drop_collection()
Example #24
0
def test_assertion():
    # fresh start
    Dataset.drop_collection()
    Assertion.drop_collection() 
    
    #create test dataset
    dataset = Dataset.create(language = 'en', name = '/data/test')
    
    #make a test assertion
    a1 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])

    #verify that it exists in the database
    assert a1.id is not None

    #make sure attributes are readable
    a1.dataset
    a1.relation
    a1.arguments
    a1.argstr
    a1.complete
    a1.context
    a1.polarity
    a1.confidence

    #make an identical test assertion
    a2 = Assertion.make('/data/test',"/rel/IsA",["/concept/test/assertion", "/concept/test/test"])

    #verify that it exists in the database
    assert a2.id is not None
    print a2.id
    print a1.id
    print a1.argstr
    
    #verify that attributes are readable
    a2.dataset
    a2.relation
    a2.arguments
    a2.argstr
    a2.complete
    a2.context
    a2.polarity
    a2.confidence
    
    #check that all checked attributes are the same
    assert a1.dataset == a2.dataset
    assert a1.relation == a2.relation
    assert a1.arguments == a2.arguments
    assert a1.polarity == a2.polarity
    assert a1.context == a2.context

    #verify that the ID's are the same
    assert (a1.id == a2.id)

    #clean up
    Dataset.drop_collection()
    Assertion.drop_collection() 
Example #25
0
def testmerge_make(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: Assertions 0-9
    DB2: Assertions 0-4
    '''
    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a = Assertion.make(
            '/data/test', '/rel/IsA',
            ['/test/assertion', 'test/test%d' % i])
        a.add_support(['/data/test/contributor/nholm'])
        a0 = Assertion.make(
            '/data/test1', '/rel/IsA',
            ['/test/assertion', 'test/test%d' % i])
        a0.add_support(['/data/test1/contributor/nholm'])

    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(5):
        a1 = Assertion.make(
            '/data/test', '/rel/IsA',
            ['/test/assertion', 'test/test%d' % i])
        a1.add_support(['/data/test/contributor/nholm'])
        a2 = Assertion.make(
            '/data/test', '/rel/HasA',
            ['/test/assertion', 'test/test%d' % i])
        a2.add_support(['/data/test/contributor/nholm'])
        a3 = Assertion.make(
            '/data/test1', '/rel/CausesDesire',
            ['/test/assertion', 'test/test%d' % i])
        a3.add_support(['/data/test1/contributor/nholm'])
Example #26
0
    def assertionExpressionLookup(self, request, obj_url):
        assertionID = request.GET['id']
        start = int(request.GET.get('start', '0'))
        limit = int(request.GET.get('limit', '10'))

        #NOTE: should return ranked by confidence score.  For now assume that they do.
        try:
            assertion = Assertion.get(assertionID)
        except DoesNotExist:
            return rc.NOT_FOUND
        cursor = Expression.objects(assertion = assertion)[start:start + limit]
        expressions = []
        while (True):
            try:
                expressions.append(str(cursor.next().id))
            except StopIteration:
                break #no more assertions within the skip/limit boundaries

        if len(expressions) == 0: #no assertions were found for the concept
            return rc.NOT_FOUND

        return "{expressions: " + str(expressions) + "}"
Example #27
0
    def assertionExpressionLookup(self, request, obj_url):
        assertionID = request.GET['id']
        start = int(request.GET.get('start', '0'))
        limit = int(request.GET.get('limit', '10'))

        #NOTE: should return ranked by confidence score.  For now assume that they do.
        try:
            assertion = Assertion.get(assertionID)
        except DoesNotExist:
            return rc.NOT_FOUND
        cursor = Expression.objects(assertion=assertion)[start:start + limit]
        expressions = []
        while (True):
            try:
                expressions.append(str(cursor.next().id))
            except StopIteration:
                break  #no more assertions within the skip/limit boundaries

        if len(expressions) == 0:  #no assertions were found for the concept
            return rc.NOT_FOUND

        return "{expressions: " + str(expressions) + "}"
Example #28
0
def test_merge3(db1='test1', db2='test2'):
    '''
    Load test assertions into the two DBs:
    DB1: assertions in one dataset
    DB2: assertions in another dataset
    '''
    print "Running test 3: "
    
    conceptdb.create_mongodb(db1)
    Assertion.drop_collection()
    Dataset.drop_collection()  
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a = Assertion.make('/data/test1','/rel/IsA',['/test/assertion','test/test%d'%i])
        a.add_support(['/data/test/contributor/nholm'])  

    
    conceptdb.create_mongodb(db2)
    Assertion.drop_collection()
    Dataset.drop_collection()
    ReasonConjunction.drop_collection()
    for i in xrange(10):
        a1 = Assertion.make('/data/test2','/rel/IsA',['/test/assertion','test/test%d'%i])  
        a1.add_support(['/data/test/contributor/nholm'])
    
    
    #testmerge_display(db1, db2)
    
    '''
    Merge the two dbs
    '''
    merge(db1, db2)
    
    '''
    Check post-merge elements, make sure they match
    '''
    testmerge_check(db1, db2)
    
    print "Finished test 3."
Example #29
0
def test_expression():

    #start clean
    Expression.drop_collection()
    Assertion.drop_collection()

    a1 = Assertion.make('/data/test', '/rel/IsA',
                        ['/concept/test/assertion', '/concept/test/test'])

    expr = Expression.make(a1, '{0} is a {1}', ['this assertion', 'test'],
                           'en')

    #check for consistency, ensure all attributes are readable
    expr.check_consistency()
    expr.assertion
    expr.text
    expr.confidence
    expr.arguments
    expr.language
    expr.frame
    expr.id

    assert expr.name == '/expression/%s' % expr.id

    # load the same assertion from the DB
    a2 = Assertion.make('/data/test', '/rel/IsA',
                        ['/concept/test/assertion', '/concept/test/test'])
    assert expr.assertion == a2

    #test static methods
    replace = Expression.replace_args(expr.frame, expr.arguments)
    assert replace == "this assertion is a test"

    #clean up
    Assertion.drop_collection()
    Expression.drop_collection()
Example #30
0
def test_expression():

    #start clean
    Expression.drop_collection()
    Assertion.drop_collection() 
    
    a1 = Assertion.make('/data/test','/rel/IsA',
                        ['/concept/test/assertion', '/concept/test/test'])
   
    expr = Expression.make(a1, '{0} is a {1}', ['this assertion', 'test'], 'en')
    
    #check for consistency, ensure all attributes are readable
    expr.check_consistency()
    expr.assertion
    expr.text
    expr.confidence
    expr.arguments
    expr.language
    expr.frame
    expr.id

    assert expr.name == '/expression/%s' % expr.id

    # load the same assertion from the DB
    a2 = Assertion.make('/data/test','/rel/IsA',
                        ['/concept/test/assertion', '/concept/test/test'])
    assert expr.assertion == a2


    #test static methods
    replace = Expression.replace_args(expr.frame, expr.arguments)
    assert replace == "this assertion is a test"

    #clean up
    Assertion.drop_collection() 
    Expression.drop_collection()
Example #31
0
    def fb_entity_from_mid(self, dset, user, polarity=1, context=None):
        ''' 
        Called when the result field is '*'
        Returns all of the results (property-value) pairs for the given mid object,
        Makes assertions of these results
        '''
        try:
            dataset = Dataset.objects.get(name=dset, language='en')
        except DoesNotExist:
            dataset = Dataset.create(name=dset, language='en')

        query = [dict(self.query_args, **{'*': {}})]

        assertionscreated = []

        # start importing from freebase
        mss = freebase.HTTPMetawebSession("http://api.freebase.com")
        try:
            results = mss.mqlread(query)
        except MetawebError:
            print 'MetawebError occurred at %s' % str(query)
            return []

        #print results[0]

        if type(results) == list:
            results = results[0]

        #for key in self.query_args.keys():
        #    initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['mid'],self.query_args[key]])
        #    initial_assertion.add_support(dataset.name + '/contributor/' + user)
        #    assertionscreated.append(initial_assertion)

        for property in [
                r for r in results if r not in self.skip_props
                and r not in self.query_args.keys()
        ]:
            #print property

            if type(results[property]) == list:
                for value in results[property]:
                    try:
                        #print self.query_args['id']
                        #print value['id']
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['mid'], value['id']])
                    except:
                        #print self.query_args['id']
                        #print value['id']
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['mid'], value['value']])

                    a.add_support(dataset.name + '/contributor/' + user)
                    assertionscreated.append(a)
            else:
                #print results[property]
                try:
                    a = Assertion.make(
                        dataset.name,
                        '/rel/freebase/has%s' % property.capitalize(),
                        [self.query_args['mid'], results[property]['value']])
                    a.add_support(dataset.name + '/contributor/' + 'nholm')
                    assertionscreated.append(a)
                except:
                    #print 'second exception'
                    #a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
                    try:
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['mid'], results[property]['id']])
                        a.add_support(dataset.name + '/contributor/' + 'nholm')
                        assertionscreated.append(a)
                    except:
                        pass
        return assertionscreated
Example #32
0
    def fb_entity_from_id(self, dset, user, polarity=1, context=None):
        ''' 
        Called when the result field is '*'
        Returns all of the results (property-value) pairs for the given id object,
        Makes assertions of these results
        '''
        try:
            dataset = Dataset.objects.get(name=dset, language='en')
        except DoesNotExist:
            dataset = Dataset.create(name=dset, language='en')

        query = [dict(self.query_args, **{'*': {}})]

        assertionscreated = []

        # start importing from freebase
        mss = freebase.HTTPMetawebSession("http://api.freebase.com")
        results = mss.mqlread(query)
        #print 'RESULTS ARE'
        #print results
        if type(results) == list:
            results = results[0]

        for key in self.query_args.keys():
            initial_assertion = Assertion.make(
                dataset.name, '/rel/freebase/has%s' % key.capitalize(),
                [self.query_args['id'], self.query_args[key]])
            initial_assertion.add_support(dataset.name + '/contributor/' +
                                          user)
            assertionscreated.append(initial_assertion)
        #print 'MADE INITIAL ASSERTION'
        # Go through all properties, excluding properties in the skip_props list and properties whose results are not of type list
        for property in [
                r for r in results if r not in self.skip_props
                and r not in self.query_args.keys()
        ]:
            # Use properties to index concepts, and only use concepts with an explicit 'mmid' field
            if type(results[property]) == list:
                for value in results[property]:
                    try:
                        #print self.query_args['id']
                        #print value['id']
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['id'], value['id']])
                    except:
                        #print self.query_args['id']
                        #print value['id']
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['id'], value['value']])

                    a.add_support(dataset.name + '/contributor/' + user)
                    assertionscreated.append(a)
            else:
                #print results[property]
                try:
                    a = Assertion.make(
                        dataset.name,
                        '/rel/freebase/has%s' % property.capitalize(),
                        [self.query_args['id'], results[property]['value']])
                    a.add_support(dataset.name + '/contributor/' + 'nholm')
                    assertionscreated.append(a)
                except:
                    #print 'second exception'
                    #a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
                    try:
                        a = Assertion.make(
                            dataset.name,
                            '/rel/freebase/has%s' % property.capitalize(),
                            [self.query_args['id'], results[property]['id']])
                        a.add_support(dataset.name + '/contributor/' + 'nholm')
                        assertionscreated.append(a)
                    except:
                        pass

        return assertionscreated
Example #33
0
    def assertionVote(self, request, obj_url):
        """Assertion vote is called whenever someone is voting on an assertion.  It can 
        be accessed in one of 2 ways: voting on an assertion identified directly by its id
        or voting on an assertion identified by its unique attributes. To add a positive vote,
        the user should make vote=1.  A negative vote is vote=-1.  Any other values will result
        in no action being taken.  

        Can be accessed through either of the following URLS:
        /api/assertionvote?dataset={dataset}&rel={relation}&concept={concept1,concept2,etc}
        &polarity={polarity}&context={context}&vote={vote}&user={username}&password={password}

        polarity and context are optional values, defaulting to polarity = 1 and context = None

        /api/assertionidvote?id={id}&vote={vote}
        """
        
        user = request.POST['user']
        password = request.POST['password']
        
         
        if obj_url.startswith('/assertionvote'):
            dataset = request.POST['dataset']
            relation = request.POST['rel']
            argstr = request.POST['concepts']
            polarity = int(request.POST.get('polarity','1'))
            context = request.POST.get('context','None')

            if context == "None":
                context = None

            
            try:
                 assertion = Assertion.objects.get(
                     dataset = dataset,
                     relation = relation,
                     argstr = argstr,
                     polarity = polarity,
                     context = context)
            except DoesNotExist:
                return rc.NOT_FOUND
        else:
            id = request.POST['id']

            try:
                assertion = Assertion.get(id)
                dataset = assertion.dataset
            except DoesNotExist:
                return rc.NOT_FOUND
        
        if User.objects.get(username=user).check_password(password):

            #the user's password is correct.  Get their reason and add
            try:
              ReasonConjunction.objects.get(target = dataset + '/contributor/' + user)
            except DoesNotExist:
              return rc.FORBIDDEN
        else:
            #incorrect password
            return rc.FORBIDDEN
         
        vote = request.POST['vote']
        if vote == "1": #vote in favor
            assertion.add_support([dataset + '/contributor/' + user]) 
        elif vote == "-1": #vote against
            assertion.add_oppose([dataset + '/contributor/' + user])
        else: #invalid vote
            return rc.BAD_REQUEST

        return assertion.serialize()
Example #34
0
    def assertionVote(self, request, obj_url):
        """Assertion vote is called whenever someone is voting on an assertion.  It can 
        be accessed in one of 2 ways: voting on an assertion identified directly by its id
        or voting on an assertion identified by its unique attributes. To add a positive vote,
        the user should make vote=1.  A negative vote is vote=-1.  Any other values will result
        in no action being taken.  

        Can be accessed through either of the following URLS:
        /api/assertionvote?dataset={dataset}&rel={relation}&concept={concept1,concept2,etc}
        &polarity={polarity}&context={context}&vote={vote}&user={username}&password={password}

        polarity and context are optional values, defaulting to polarity = 1 and context = None

        /api/assertionidvote?id={id}&vote={vote}
        """

        user = request.POST['user']
        password = request.POST['password']

        if obj_url.startswith('/assertionvote'):
            dataset = request.POST['dataset']
            relation = request.POST['rel']
            argstr = request.POST['concepts']
            polarity = int(request.POST.get('polarity', '1'))
            context = request.POST.get('context', 'None')

            if context == "None":
                context = None

            try:
                assertion = Assertion.objects.get(dataset=dataset,
                                                  relation=relation,
                                                  argstr=argstr,
                                                  polarity=polarity,
                                                  context=context)
            except DoesNotExist:
                return rc.NOT_FOUND
        else:
            id = request.POST['id']

            try:
                assertion = Assertion.get(id)
                dataset = assertion.dataset
            except DoesNotExist:
                return rc.NOT_FOUND

        if User.objects.get(username=user).check_password(password):

            #the user's password is correct.  Get their reason and add
            try:
                ReasonConjunction.objects.get(target=dataset +
                                              '/contributor/' + user)
            except DoesNotExist:
                return rc.FORBIDDEN
        else:
            #incorrect password
            return rc.FORBIDDEN

        vote = request.POST['vote']
        if vote == "1":  #vote in favor
            assertion.add_support([dataset + '/contributor/' + user])
        elif vote == "-1":  #vote against
            assertion.add_oppose([dataset + '/contributor/' + user])
        else:  #invalid vote
            return rc.BAD_REQUEST

        return assertion.serialize()
Example #35
0
import conceptdb
from conceptdb.assertion import Assertion, Expression, Sentence
from conceptdb.metadata import Dataset
from conceptdb.justify import ReasonConjunction

conceptdb.connect_to_mongodb('test')

#clean out whatever was in test before
Assertion.drop_collection()
Dataset.drop_collection()
Expression.drop_collection()
ReasonConjunction.drop_collection()
Sentence.drop_collection()

#make a new dataset
d = Dataset.make('/data/test', 'en')

#make a couple of assertions
a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test'])
a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test'])

#add expressions to them
e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language)
e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language)

#connect them to sentences
a1.connect_to_sentence(d, 'This assertion is a test.')
a2.connect_to_sentence(d, 'This database is a test.')

#make a reason for myself
r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'],
Example #36
0
def import_assertions(lang, skip=0):
    dataset = Dataset.make(DATASET_ROOT + lang, lang)
    generalize_reason = dataset.get_reason('/rule/generalize')
    justify(generalize_reason, [dataset.get_root_reason()], 0.1)

    assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\
      .select_related('concept1', 'concept2', 'relation', 'language')[skip:]
    for assertion in assertions:
        relation = RELATION_ROOT + assertion.relation.name
        concept_names = [assertion.concept1.text, assertion.concept2.text]
        concepts = [CONCEPT_ROOT + c for c in concept_names]
        context = None
        if -5 < assertion.frequency < 5:
            context = '/concept/frequency/en/sometimes'
        raws = assertion.rawassertion_set.all().select_related(
            'surface1', 'surface2', 'frame', 'sentence', 'sentence__creator',
            'sentence__activity')

        newassertion = Assertion.make(dataset,
                                      relation,
                                      concepts,
                                      polarity=assertion.polarity,
                                      context=context)
        newassertion.save()

        sent_contributors = set()
        support_votes = assertion.votes.filter(vote=1)
        oppose_votes = assertion.votes.filter(vote=-1)
        for vote in support_votes:
            voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username)
            if voter not in sent_contributors:
                newassertion.add_support([voter])
        for vote in oppose_votes:
            voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username)
            newassertion.add_oppose([voter])

        for raw in raws:
            if raw.score > 0:
                frametext = raw.frame.text.replace('{1}', '{0}').replace(
                    '{2}', '{1}').replace('{%}', '')
                expr = newassertion.make_expression(
                    frametext, [raw.surface1.text, raw.surface2.text],
                    assertion.language.id)
                support_votes = raw.votes.filter(vote=1).select_related('user')
                oppose_votes = raw.votes.filter(vote=-1).select_related('user')
                for vote in support_votes:
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT +
                                               vote.user.username)
                    expr.add_support([voter])
                for vote in oppose_votes:
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT +
                                               vote.user.username)
                    expr.add_oppose([voter])
                expr.save()

                sent = raw.sentence
                if sent.score > 0:
                    activity = sent.activity.name
                    act_reason = dataset.get_reason(ACTIVITY_ROOT +
                                                    activity.replace(' ', '_'))
                    voter = dataset.get_reason(CONTRIBUTOR_ROOT +
                                               vote.user.username)

                    sent_contributors.add(voter)
                    justification = [act_reason, voter]
                    newassertion.connect_to_sentence(dataset, sent.text,
                                                     justification)

        newassertion.make_generalizations(generalize_reason)
        log.info(newassertion)
import conceptdb
from conceptdb.assertion import Assertion, Expression, Sentence
from conceptdb.metadata import Dataset
from conceptdb.justify import ReasonConjunction

conceptdb.connect_to_mongodb('test')

#clean out whatever was in test before
Assertion.drop_collection()
Dataset.drop_collection()
Expression.drop_collection()
ReasonConjunction.drop_collection()
Sentence.drop_collection()

#make a new dataset
d = Dataset.make('/data/test', 'en')

#make a couple of assertions
a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test'])
a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test'])

#add expressions to them
e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language)
e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language)

#connect them to sentences
a1.connect_to_sentence(d, 'This assertion is a test.')
a2.connect_to_sentence(d, 'This database is a test.')

#make a reason for myself
r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'], 0.75, True)
Example #38
0
def testmerge_check(db1='test1', db2='test2', dataset=None):
    '''
    Check post-merge elements, make sure they match
    '''

    if dataset==None:
        print "Testing Assertion objects"
        conceptdb.connect_to_mongodb(db1)
        db1_assertions = Assertion.objects
        conceptdb.connect_to_mongodb(db2)
        db2_assertions = Assertion.objects
        

        for db1_a in [a1 for a1 in list(db1_assertions) if a1 not in list(db2_assertions)]:
            check = False
            for db2_check in list(db2_assertions):
                if assertion_check(db1_a, db2_check):
                    check = True        
            assert check==True
        
        for db2_a in [a2 for a2 in list(db2_assertions) if a2 not in list(db1_assertions)]:
            check = False
            for db1_check in list(db1_assertions):
                if assertion_check(db2_a, db1_check):
                    check = True
            assert check==True
        
        print "Assertion test PASSED"
        
        print "Testing ReasonConjunction objects"
        conceptdb.connect_to_mongodb(db1)
        #print "After the merge, db %s has the following assertions: "%db1
        for a1 in Assertion.objects:
            #print "assertion: %s"%a1
            #print "     confidence score: %s"%a1.confidence
            for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
                #print "     reason: %s"%r1.factors
                assert r1.target == a1.name

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
    
        conceptdb.connect_to_mongodb(db2)
        #print "After the merge, db %s has the following assertions: "%db2
        for a2 in Assertion.objects:
            #print "assertion: %s"%a2
            #print "     confidence score: %s"%a2.confidence
            for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
                #print "     reason: %s"%r2.factors
                assert r2.target == a2.name
        
        print "ReasonConjunction test PASSED"

        Assertion.drop_collection() 
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
    else:
        print "Testing Assertion objects"
        conceptdb.connect_to_mongodb(db1)
        db1_assertions = Assertion.objects.filter(dataset=dataset)
        conceptdb.connect_to_mongodb(db2)
        db2_assertions = Assertion.objects.filter(dataset=dataset)
        
        for db1_a in [a1 for a1 in list(db1_assertions) if a1 not in list(db2_assertions)]:
            check = False
            for db2_check in list(db2_assertions):
                if assertion_check(db1_a, db2_check):
                    check = True        
            assert check==True
        
        for db2_a in [a2 for a2 in list(db2_assertions) if a2 not in list(db1_assertions)]:
            check = False
            for db1_check in list(db1_assertions):
                if assertion_check(db2_a, db1_check):
                    check = True
            assert check==True
        
        print "Assertion test PASSED"
        
        
        print "Testing ReasonConjunction objects"
        conceptdb.connect_to_mongodb(db1)
        #print "After the merge, db %s has the following assertions: "%db1
        for a1 in Assertion.objects.filter(dataset=dataset):
            #print "assertion: %s"%a1
            #print "     confidence score: %s"%a1.confidence
            for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
                #print "     reason: %s"%r1.factors
                assert r1.target == a1.name

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
    
        conceptdb.connect_to_mongodb(db2)
        #print "After the merge, db %s has the following assertions: "%db2
        for a2 in Assertion.objects.filter(dataset=dataset):
            #print "assertion: %s"%a2
            #print "     confidence score: %s"%a2.confidence
            for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
                #print "     reason: %s"%r2.factors
                assert r2.target == a2.name
        
        print "ReasonConjunction test PASSED"

        Assertion.drop_collection() 
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
Example #39
0
def testmerge_check(db1='test1', db2='test2', dataset=None):
    '''
    Check post-merge elements, make sure they match
    '''

    if dataset == None:
        print "Testing Assertion objects"
        conceptdb.connect_to_mongodb(db1)
        db1_assertions = Assertion.objects
        conceptdb.connect_to_mongodb(db2)
        db2_assertions = Assertion.objects

        for db1_a in [
                a1 for a1 in list(db1_assertions)
                if a1 not in list(db2_assertions)
        ]:
            check = False
            for db2_check in list(db2_assertions):
                if assertion_check(db1_a, db2_check):
                    check = True
            assert check == True

        for db2_a in [
                a2 for a2 in list(db2_assertions)
                if a2 not in list(db1_assertions)
        ]:
            check = False
            for db1_check in list(db1_assertions):
                if assertion_check(db2_a, db1_check):
                    check = True
            assert check == True

        print "Assertion test PASSED"

        print "Testing ReasonConjunction objects"
        conceptdb.connect_to_mongodb(db1)
        #print "After the merge, db %s has the following assertions: "%db1
        for a1 in Assertion.objects:
            #print "assertion: %s"%a1
            #print "     confidence score: %s"%a1.confidence
            for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
                #print "     reason: %s"%r1.factors
                assert r1.target == a1.name

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()

        conceptdb.connect_to_mongodb(db2)
        #print "After the merge, db %s has the following assertions: "%db2
        for a2 in Assertion.objects:
            #print "assertion: %s"%a2
            #print "     confidence score: %s"%a2.confidence
            for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
                #print "     reason: %s"%r2.factors
                assert r2.target == a2.name

        print "ReasonConjunction test PASSED"

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
    else:
        print "Testing Assertion objects"
        conceptdb.connect_to_mongodb(db1)
        db1_assertions = Assertion.objects.filter(dataset=dataset)
        conceptdb.connect_to_mongodb(db2)
        db2_assertions = Assertion.objects.filter(dataset=dataset)

        for db1_a in [
                a1 for a1 in list(db1_assertions)
                if a1 not in list(db2_assertions)
        ]:
            check = False
            for db2_check in list(db2_assertions):
                if assertion_check(db1_a, db2_check):
                    check = True
            assert check == True

        for db2_a in [
                a2 for a2 in list(db2_assertions)
                if a2 not in list(db1_assertions)
        ]:
            check = False
            for db1_check in list(db1_assertions):
                if assertion_check(db2_a, db1_check):
                    check = True
            assert check == True

        print "Assertion test PASSED"

        print "Testing ReasonConjunction objects"
        conceptdb.connect_to_mongodb(db1)
        #print "After the merge, db %s has the following assertions: "%db1
        for a1 in Assertion.objects.filter(dataset=dataset):
            #print "assertion: %s"%a1
            #print "     confidence score: %s"%a1.confidence
            for r1 in list(ReasonConjunction.objects.filter(target=a1.name)):
                #print "     reason: %s"%r1.factors
                assert r1.target == a1.name

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()

        conceptdb.connect_to_mongodb(db2)
        #print "After the merge, db %s has the following assertions: "%db2
        for a2 in Assertion.objects.filter(dataset=dataset):
            #print "assertion: %s"%a2
            #print "     confidence score: %s"%a2.confidence
            for r2 in list(ReasonConjunction.objects.filter(target=a2.name)):
                #print "     reason: %s"%r2.factors
                assert r2.target == a2.name

        print "ReasonConjunction test PASSED"

        Assertion.drop_collection()
        Dataset.drop_collection()
        ReasonConjunction.drop_collection()
Example #40
0
 def fb_entity_from_id(self, dset, user, polarity=1, context=None):
     ''' 
     Called when the result field is '*'
     Returns all of the results (property-value) pairs for the given id object,
     Makes assertions of these results
     '''
     try:
         dataset = Dataset.objects.get(name=dset, language='en')
     except DoesNotExist:
         dataset = Dataset.create(name=dset, language='en')
 
     query = [dict(self.query_args,**{'*':{}})]
     
     assertionscreated = []
     
     # start importing from freebase
     mss = freebase.HTTPMetawebSession("http://api.freebase.com")
     results = mss.mqlread(query)
     #print 'RESULTS ARE'
     #print results
     if type(results)==list:
         results = results[0]
     
     for key in self.query_args.keys():
         initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['id'],self.query_args[key]])
         initial_assertion.add_support(dataset.name + '/contributor/' + user)
         assertionscreated.append(initial_assertion)
     #print 'MADE INITIAL ASSERTION'
     # Go through all properties, excluding properties in the skip_props list and properties whose results are not of type list  
     for property in [r for r in results if r not in self.skip_props and r not in self.query_args.keys()]:
         # Use properties to index concepts, and only use concepts with an explicit 'mmid' field
         if type(results[property])==list:
             for value in results[property]:
                 try:
                     #print self.query_args['id']
                     #print value['id']
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],value['id']])
                 except:
                     #print self.query_args['id']
                     #print value['id']
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],value['value']])
             
                 a.add_support(dataset.name + '/contributor/' + user)
                 assertionscreated.append(a)
         else:
             #print results[property]
             try:
                 a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],results[property]['value']])
                 a.add_support(dataset.name + '/contributor/' + 'nholm')
                 assertionscreated.append(a)
             except:
                 #print 'second exception'
                 #a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
                 try:
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['id'],results[property]['id']])
                     a.add_support(dataset.name + '/contributor/' + 'nholm')
                     assertionscreated.append(a)
                 except:
                     pass
         
     return assertionscreated
Example #41
0
 def fb_entity_from_mid(self, dset, user, polarity=1, context=None):
     ''' 
     Called when the result field is '*'
     Returns all of the results (property-value) pairs for the given mid object,
     Makes assertions of these results
     '''
     try:
         dataset = Dataset.objects.get(name=dset, language='en')
     except DoesNotExist:
         dataset = Dataset.create(name=dset, language='en')
 
     query = [dict(self.query_args,**{'*':{}})]
     
     assertionscreated = []
     
     # start importing from freebase
     mss = freebase.HTTPMetawebSession("http://api.freebase.com")
     try:
         results = mss.mqlread(query)
     except MetawebError:
         print 'MetawebError occurred at %s'%str(query)
         return []
     
     
     #print results[0]
     
     if type(results)==list:
         results = results[0]
     
     #for key in self.query_args.keys():
     #    initial_assertion = Assertion.make(dataset.name, '/rel/freebase/has%s'%key.capitalize(), [self.query_args['mid'],self.query_args[key]])
     #    initial_assertion.add_support(dataset.name + '/contributor/' + user)
     #    assertionscreated.append(initial_assertion)
     
     for property in [r for r in results if r not in self.skip_props and r not in self.query_args.keys()]:
         #print property
         
         if type(results[property])==list:
             for value in results[property]:
                 try:
                     #print self.query_args['id']
                     #print value['id']
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],value['id']])
                 except:
                     #print self.query_args['id']
                     #print value['id']
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],value['value']])
             
                 a.add_support(dataset.name + '/contributor/' + user)
                 assertionscreated.append(a)
         else:
             #print results[property]
             try:
                 a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],results[property]['value']])
                 a.add_support(dataset.name + '/contributor/' + 'nholm')
                 assertionscreated.append(a)
             except:
                 #print 'second exception'
                 #a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mmid'],results[property]['mmid']])
                 try:
                     a = Assertion.make(dataset.name, '/rel/freebase/has%s'%property.capitalize(), [self.query_args['mid'],results[property]['id']])
                     a.add_support(dataset.name + '/contributor/' + 'nholm')
                     assertionscreated.append(a)
                 except:
                     pass
     return assertionscreated