def import_activities(lang): dataset = Dataset.make(DATASET_ROOT + lang, lang) site = dataset.get_reason('/site/omcs') for act in Activity.objects.iterator(): newact = dataset.get_reason(ACTIVITY_ROOT + act.name.replace(' ', '_')) justify(newact, [site], 1.0) log.info(newact)
def test_dataset(): #start with an empty collection Dataset.drop_collection() dataset = Dataset.make(language='en', name='/data/test') # Make sure it was saved to the database assert dataset.id is not None # Make sure its attributes are readable dataset.language dataset.name #test methods reason = dataset.get_root_reason() assert '/data/test/root' == reason dataset.nl dataset.check_consistency() # Clean up by dropping the Dataset collection Dataset.drop_collection()
def import_activities(lang): dataset = Dataset.make(DATASET_ROOT+lang, lang) site = dataset.get_reason('/site/omcs') for act in Activity.objects.iterator(): newact = dataset.get_reason(ACTIVITY_ROOT+act.name.replace(' ', '_')) justify(newact, [site], 1.0) log.info(newact)
def import_assertions(lang, skip=0): dataset = Dataset.make(DATASET_ROOT+lang, lang) generalize_reason = dataset.get_reason('/rule/generalize') justify(generalize_reason, [dataset.get_root_reason()], 0.1) assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\ .select_related('concept1', 'concept2', 'relation', 'language')[skip:] for assertion in assertions: relation = RELATION_ROOT + assertion.relation.name concept_names = [assertion.concept1.text, assertion.concept2.text] concepts = [CONCEPT_ROOT+c for c in concept_names] context = None if -5 < assertion.frequency < 5: context = '/concept/frequency/en/sometimes' raws = assertion.rawassertion_set.all().select_related('surface1', 'surface2', 'frame', 'sentence', 'sentence__creator', 'sentence__activity') newassertion = Assertion.make(dataset, relation, concepts, polarity = assertion.polarity, context=context) newassertion.save() sent_contributors = set() support_votes = assertion.votes.filter(vote=1) oppose_votes = assertion.votes.filter(vote=-1) for vote in support_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username) if voter not in sent_contributors: newassertion.add_support([voter]) for vote in oppose_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username) newassertion.add_oppose([voter]) for raw in raws: if raw.score > 0: frametext = raw.frame.text.replace('{1}','{0}').replace('{2}','{1}').replace('{%}','') expr = newassertion.make_expression(frametext, [raw.surface1.text, raw.surface2.text], assertion.language.id) support_votes = raw.votes.filter(vote=1).select_related('user') oppose_votes = raw.votes.filter(vote=-1).select_related('user') for vote in support_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username) expr.add_support([voter]) for vote in oppose_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username) expr.add_oppose([voter]) expr.save() sent = raw.sentence if sent.score > 0: activity = sent.activity.name act_reason = dataset.get_reason(ACTIVITY_ROOT+activity.replace(' ', '_')) voter = dataset.get_reason(CONTRIBUTOR_ROOT+vote.user.username) sent_contributors.add(voter) justification = [act_reason, voter] newassertion.connect_to_sentence(dataset, sent.text, justification) newassertion.make_generalizations(generalize_reason) log.info(newassertion)
def import_contributors(lang): dataset = Dataset.make(DATASET_ROOT+lang, lang) root = dataset.get_root_reason() site = dataset.get_reason('/site/omcs') justify(site, [root], 1.0) for user in User.objects.iterator(): contributor = dataset.get_reason(CONTRIBUTOR_ROOT+user.username) justify(contributor, [site], 0.5) log.info(contributor)
def import_contributors(lang): dataset = Dataset.make(DATASET_ROOT + lang, lang) root = dataset.get_root_reason() site = dataset.get_reason('/site/omcs') justify(site, [root], 1.0) for user in User.objects.iterator(): contributor = dataset.get_reason(CONTRIBUTOR_ROOT + user.username) justify(contributor, [site], 0.5) log.info(contributor)
def import_sentence(sent): dataset = Dataset.make(DATASET_ROOT+sent.language.id, sent.language.id) activity = sent.activity.name root = dataset.get_root_reason() site = root.derived_reason('/site/omcs') act_reason = site.derived_reason(ACTIVITY_ROOT+activity.replace(' ', '_')) contrib_reason = site.derived_reason(CONTRIBUTOR_ROOT+sent.creator.username) justification = [act_reason, contrib_reason] newsent = assertion.Sentence.make(dataset, sent.text, justification) log.info(str(newsent))
def import_sentence(sent): dataset = Dataset.make(DATASET_ROOT + sent.language.id, sent.language.id) activity = sent.activity.name root = dataset.get_root_reason() site = root.derived_reason('/site/omcs') act_reason = site.derived_reason(ACTIVITY_ROOT + activity.replace(' ', '_')) contrib_reason = site.derived_reason(CONTRIBUTOR_ROOT + sent.creator.username) justification = [act_reason, contrib_reason] newsent = assertion.Sentence.make(dataset, sent.text, justification) log.info(str(newsent))
import conceptdb from conceptdb.assertion import Assertion, Expression, Sentence from conceptdb.metadata import Dataset from conceptdb.justify import ReasonConjunction conceptdb.connect_to_mongodb('test') #clean out whatever was in test before Assertion.drop_collection() Dataset.drop_collection() Expression.drop_collection() ReasonConjunction.drop_collection() Sentence.drop_collection() #make a new dataset d = Dataset.make('/data/test', 'en') #make a couple of assertions a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test']) a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test']) #add expressions to them e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language) e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language) #connect them to sentences a1.connect_to_sentence(d, 'This assertion is a test.') a2.connect_to_sentence(d, 'This database is a test.') #make a reason for myself r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'], 0.75, True)
def import_assertions(lang, skip=0): dataset = Dataset.make(DATASET_ROOT + lang, lang) generalize_reason = dataset.get_reason('/rule/generalize') justify(generalize_reason, [dataset.get_root_reason()], 0.1) assertions = OldAssertion.objects.filter(score__gt=0, language__id=lang)\ .select_related('concept1', 'concept2', 'relation', 'language')[skip:] for assertion in assertions: relation = RELATION_ROOT + assertion.relation.name concept_names = [assertion.concept1.text, assertion.concept2.text] concepts = [CONCEPT_ROOT + c for c in concept_names] context = None if -5 < assertion.frequency < 5: context = '/concept/frequency/en/sometimes' raws = assertion.rawassertion_set.all().select_related( 'surface1', 'surface2', 'frame', 'sentence', 'sentence__creator', 'sentence__activity') newassertion = Assertion.make(dataset, relation, concepts, polarity=assertion.polarity, context=context) newassertion.save() sent_contributors = set() support_votes = assertion.votes.filter(vote=1) oppose_votes = assertion.votes.filter(vote=-1) for vote in support_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username) if voter not in sent_contributors: newassertion.add_support([voter]) for vote in oppose_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username) newassertion.add_oppose([voter]) for raw in raws: if raw.score > 0: frametext = raw.frame.text.replace('{1}', '{0}').replace( '{2}', '{1}').replace('{%}', '') expr = newassertion.make_expression( frametext, [raw.surface1.text, raw.surface2.text], assertion.language.id) support_votes = raw.votes.filter(vote=1).select_related('user') oppose_votes = raw.votes.filter(vote=-1).select_related('user') for vote in support_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username) expr.add_support([voter]) for vote in oppose_votes: voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username) expr.add_oppose([voter]) expr.save() sent = raw.sentence if sent.score > 0: activity = sent.activity.name act_reason = dataset.get_reason(ACTIVITY_ROOT + activity.replace(' ', '_')) voter = dataset.get_reason(CONTRIBUTOR_ROOT + vote.user.username) sent_contributors.add(voter) justification = [act_reason, voter] newassertion.connect_to_sentence(dataset, sent.text, justification) newassertion.make_generalizations(generalize_reason) log.info(newassertion)
import conceptdb from conceptdb.assertion import Assertion, Expression, Sentence from conceptdb.metadata import Dataset from conceptdb.justify import ReasonConjunction conceptdb.connect_to_mongodb('test') #clean out whatever was in test before Assertion.drop_collection() Dataset.drop_collection() Expression.drop_collection() ReasonConjunction.drop_collection() Sentence.drop_collection() #make a new dataset d = Dataset.make('/data/test', 'en') #make a couple of assertions a1 = Assertion.make(d, '/rel/IsA', ['/concept/assertion', '/concept/test']) a2 = Assertion.make(d, '/rel/IsA', ['/concept/database', '/concept/test']) #add expressions to them e1 = a1.make_expression('{0} is a {1}', a1.arguments, d.language) e2 = a2.make_expression('{0} is a {1}', a2.arguments, d.language) #connect them to sentences a1.connect_to_sentence(d, 'This assertion is a test.') a2.connect_to_sentence(d, 'This database is a test.') #make a reason for myself r = ReasonConjunction.make('/data/test/contributor/elhawk', ['user_factor'],