def test_filter_namespace_concepts_list():
def make_statement(a, b):
return Influence(Concept(a, db_refs={'UN': [(a, 1.0)]}),
Concept(b, db_refs={'UN': [(b, 1.0)]}))
stmts = [
make_statement('UN/entities/human/education',
'UN/entities/human/food/food_security'),
make_statement('UN/entities/human/fishery',
'UN/entities/human/government')
]
fs = ac.filter_by_db_refs(stmts,
'UN', ['education'],
'one',
match_suffix=True)
assert [stmts[0]] == fs, fs
fs = ac.filter_by_db_refs(stmts,
'UN', ['education', 'fishery'],
'one',
match_suffix=True)
assert stmts == fs, fs
fs = ac.filter_by_db_refs(stmts,
'UN', ['fishery', 'government'],
'all',
match_suffix=True)
assert [stmts[1]] == fs, fs
def test_filter_namespace_concepts_simple():
def make_statement(a, b):
return Influence(Event(Concept(a, db_refs={'TEXT': a})),
Event(Concept(b, db_refs={'TEXT': b})))
stmts = [make_statement('education', 'thinking'),
make_statement('doubt', 'government')]
fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'one')
assert [stmts[0]] == fs, fs
fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'one',
invert=True)
assert stmts == fs, fs
fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'all',
invert=True)
assert [stmts[1]] == fs, fs
fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'all')
assert not fs, fs
def test_filter_namespace_concepts_list():
def make_statement(a, b):
return Influence(Event(Concept(a, db_refs={'UN': [(a, 1.0)]})),
Event(Concept(b, db_refs={'UN': [(b, 1.0)]})))
stmts = [make_statement('UN/entities/human/education',
'UN/entities/human/food/food_security'),
make_statement('UN/entities/human/fishery',
'UN/entities/human/government')]
fs = ac.filter_by_db_refs(stmts, 'UN', ['education'], 'one',
match_suffix=True)
assert [stmts[0]] == fs, fs
fs = ac.filter_by_db_refs(stmts, 'UN', ['education', 'fishery'],
'one', match_suffix=True)
assert stmts == fs, fs
fs = ac.filter_by_db_refs(stmts, 'UN',
['fishery', 'government'], 'all',
match_suffix=True)
assert [stmts[1]] == fs, fs
def filter_groundings(stmts):
with open('groundings_to_exclude.txt', 'r') as f:
groundings_to_exclude = [l.strip() for l in f.readlines()]
stmts = ac.filter_by_db_refs(stmts,
'WM',
groundings_to_exclude,
'all',
invert=True)
return stmts
def filter_groundings(stmts):
excl_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
os.pardir, 'resources',
'groundings_to_exclude.txt')
with open(excl_file, 'r') as f:
groundings_to_exclude = [l.strip() for l in f.readlines()]
stmts = ac.filter_by_db_refs(stmts,
'WM',
groundings_to_exclude,
'all',
invert=True)
return stmts
def run_preassembly(statements, hierarchies):
print('%d total statements' % len(statements))
# Filter to grounded only
statements = map_onto(statements)
ac.dump_statements(statements, 'pi_mtg_demo_unfiltered.pkl')
statements = ac.filter_grounded_only(statements, score_threshold=0.7)
#statements = ac.filter_by_db_refs(statements, 'UN',
# ['conflict', 'food_security', 'precipitation'], policy='one',
# match_suffix=True)
statements = ac.filter_by_db_refs(
statements,
'UN', [
'conflict', 'food_security', 'flooding', 'food_production',
'human_migration', 'drought', 'food_availability', 'market',
'food_insecurity'
],
policy='all',
match_suffix=True)
assume_polarity(statements)
statements = filter_has_polarity(statements)
# Make a Preassembler with the Eidos and TRIPS ontology
pa = Preassembler(hierarchies, statements)
# Make a BeliefEngine and run combine duplicates
be = BeliefEngine()
unique_stmts = pa.combine_duplicates()
print('%d unique statements' % len(unique_stmts))
be.set_prior_probs(unique_stmts)
# Run combine related
related_stmts = pa.combine_related(return_toplevel=False)
be.set_hierarchy_probs(related_stmts)
#related_stmts = ac.filter_belief(related_stmts, 0.8)
# Filter to top-level Statements
top_stmts = ac.filter_top_level(related_stmts)
pa.stmts = top_stmts
print('%d top-level statements' % len(top_stmts))
conflicts = pa.find_contradicts()
top_stmts = remove_contradicts(top_stmts, conflicts)
ac.dump_statements(top_stmts, 'pi_mtg_demo.pkl')
return top_stmts