from csc_utils.batch import queryset_foreach from conceptnet.models import Sentence, Assertion, RawAssertion queryset_foreach(Assertion.objects.all(), lambda x: x.update_score(), batch_size=100) queryset_foreach(RawAssertion.objects.all(), lambda x: x.update_score(), batch_size=100) # queryset_foreach(Sentence.objects.exclude(language__id='en'), lambda x: x.update_score(), batch_size=100)
def update_votes():
queryset_foreach(RawAssertion,
lambda x: fix_raw_assertion_vote(x),
batch_size=100)
def update_scores():
queryset_foreach(Assertion, lambda x: x.update_score(), batch_size=100)
queryset_foreach(RawAssertion, lambda x: x.update_score(), batch_size=100)
relname = raw.frame.relation.name
if relname == 'ConceptuallyRelatedTo':
relname = 'RelatedTo'
if polarity > 0:
relation = normalize_uri('/r/'+relname)
else:
relation = normalize_uri('/r/Not'+relname)
dataset = normalize_uri('/d/nadya.jp')
score = raw.score
sources = [([activity_node], score/5.)]
for source_list, weight in sources:
if 'commons2_reject' in ' '.join(source_list):
weight = -1
start = make_concept_uri(startText, lang)
end = make_concept_uri(endText, lang)
edge = make_edge(relation, start, end, dataset, LICENSE, source_list, '/ctx/all', frame_text, weight=weight)
writer.write(edge)
except Exception:
import traceback
traceback.print_exc()
if __name__ == '__main__':
writer = MultiWriter('nadya.jp')
queryset_foreach(RawAssertion.objects.filter(), lambda item: handle_raw_assertion(item, writer))
writer.close()
sources = [([creator_node, activity_node], 1)]
for vote in raw.votes.all():
sources.append(([normalize_uri('/s/contributor/omcs/'+vote.user.username),
normalize_uri(u'/s/activity/omcs/vote')], vote.vote))
for source_list, weight in sources:
bad = False
if 'commons2_reject' in ' '.join(source_list):
weight = -1
start = make_concept_uri(startText, lang)
end = make_concept_uri(endText, lang)
if 'bedume' in ' '.join(source_list):
for flagged in BEDUME_FLAGGED_CONCEPTS + BEDUME_FLAGGED_PLACES:
check = '/'+flagged.replace(' ', '_')
if start.endswith(check) or end.endswith(check):
bad = True
print "flagged:", str(raw)
break
if not bad:
edge = make_edge(relation, start, end, dataset, LICENSE, source_list, '/ctx/all', frame_text, weight=weight)
writer.write(edge)
except Exception:
import traceback
traceback.print_exc()
if __name__ == '__main__':
writer = MultiWriter('conceptnet4')
queryset_foreach(RawAssertion.objects.filter(), lambda item: handle_raw_assertion(item, writer))
writer.close()
arg1 = GRAPH.get_or_create_concept('ja', JA.normalize(raw.text1))
arg2 = GRAPH.get_or_create_concept('ja', JA.normalize(raw.text2))
else:
nlp = simplenlp.get(lang)
arg1 = GRAPH.get_or_create_concept(lang, nlp.normalize(raw.text1))
arg2 = GRAPH.get_or_create_concept(lang, nlp.normalize(raw.text2))
if polarity > 0:
relation = GRAPH.get_or_create_relation(raw.frame.relation.name)
else:
relation = GRAPH.get_or_create_relation('Not'+raw.frame.relation.name)
assertion = GRAPH.get_or_create_assertion(
relation, [arg1, arg2],
{'dataset': 'conceptnet/4/'+lang, 'license': 'CC-By', 'normalized': True}
)
for vote in raw.votes.all():
voter = GRAPH.get_or_create_node(
u'/source/contributor/omcs/'+vote.user.username
)
GRAPH.justify(OMCS, voter)
GRAPH.justify(voter, raw_assertion, weight=vote.vote)
GRAPH.derive_normalized(raw_assertion, assertion)
print assertion
except Exception:
import traceback
traceback.print_exc()
if __name__ == '__main__':
queryset_foreach(RawAssertion, put_raw_assertion_in_graph)
def update_scores():
queryset_foreach(Assertion, lambda x: x.update_score(),
batch_size=100)
queryset_foreach(RawAssertion, lambda x: x.update_score(),
batch_size=100)
def update_votes():
queryset_foreach(RawAssertion, lambda x: fix_raw_assertion_vote(x), batch_size=100)
