def test_filter_namespace_concepts_list(): def make_statement(a, b): return Influence(Concept(a, db_refs={'UN': [(a, 1.0)]}), Concept(b, db_refs={'UN': [(b, 1.0)]})) stmts = [ make_statement('UN/entities/human/education', 'UN/entities/human/food/food_security'), make_statement('UN/entities/human/fishery', 'UN/entities/human/government') ] fs = ac.filter_by_db_refs(stmts, 'UN', ['education'], 'one', match_suffix=True) assert [stmts[0]] == fs, fs fs = ac.filter_by_db_refs(stmts, 'UN', ['education', 'fishery'], 'one', match_suffix=True) assert stmts == fs, fs fs = ac.filter_by_db_refs(stmts, 'UN', ['fishery', 'government'], 'all', match_suffix=True) assert [stmts[1]] == fs, fs
def test_filter_namespace_concepts_simple(): def make_statement(a, b): return Influence(Event(Concept(a, db_refs={'TEXT': a})), Event(Concept(b, db_refs={'TEXT': b}))) stmts = [make_statement('education', 'thinking'), make_statement('doubt', 'government')] fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'one') assert [stmts[0]] == fs, fs fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'one', invert=True) assert stmts == fs, fs fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'all', invert=True) assert [stmts[1]] == fs, fs fs = ac.filter_by_db_refs(stmts, 'TEXT', ['education'], 'all') assert not fs, fs
def test_filter_namespace_concepts_list(): def make_statement(a, b): return Influence(Event(Concept(a, db_refs={'UN': [(a, 1.0)]})), Event(Concept(b, db_refs={'UN': [(b, 1.0)]}))) stmts = [make_statement('UN/entities/human/education', 'UN/entities/human/food/food_security'), make_statement('UN/entities/human/fishery', 'UN/entities/human/government')] fs = ac.filter_by_db_refs(stmts, 'UN', ['education'], 'one', match_suffix=True) assert [stmts[0]] == fs, fs fs = ac.filter_by_db_refs(stmts, 'UN', ['education', 'fishery'], 'one', match_suffix=True) assert stmts == fs, fs fs = ac.filter_by_db_refs(stmts, 'UN', ['fishery', 'government'], 'all', match_suffix=True) assert [stmts[1]] == fs, fs
def filter_groundings(stmts): with open('groundings_to_exclude.txt', 'r') as f: groundings_to_exclude = [l.strip() for l in f.readlines()] stmts = ac.filter_by_db_refs(stmts, 'WM', groundings_to_exclude, 'all', invert=True) return stmts
def filter_groundings(stmts): excl_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'resources', 'groundings_to_exclude.txt') with open(excl_file, 'r') as f: groundings_to_exclude = [l.strip() for l in f.readlines()] stmts = ac.filter_by_db_refs(stmts, 'WM', groundings_to_exclude, 'all', invert=True) return stmts
def run_preassembly(statements, hierarchies): print('%d total statements' % len(statements)) # Filter to grounded only statements = map_onto(statements) ac.dump_statements(statements, 'pi_mtg_demo_unfiltered.pkl') statements = ac.filter_grounded_only(statements, score_threshold=0.7) #statements = ac.filter_by_db_refs(statements, 'UN', # ['conflict', 'food_security', 'precipitation'], policy='one', # match_suffix=True) statements = ac.filter_by_db_refs( statements, 'UN', [ 'conflict', 'food_security', 'flooding', 'food_production', 'human_migration', 'drought', 'food_availability', 'market', 'food_insecurity' ], policy='all', match_suffix=True) assume_polarity(statements) statements = filter_has_polarity(statements) # Make a Preassembler with the Eidos and TRIPS ontology pa = Preassembler(hierarchies, statements) # Make a BeliefEngine and run combine duplicates be = BeliefEngine() unique_stmts = pa.combine_duplicates() print('%d unique statements' % len(unique_stmts)) be.set_prior_probs(unique_stmts) # Run combine related related_stmts = pa.combine_related(return_toplevel=False) be.set_hierarchy_probs(related_stmts) #related_stmts = ac.filter_belief(related_stmts, 0.8) # Filter to top-level Statements top_stmts = ac.filter_top_level(related_stmts) pa.stmts = top_stmts print('%d top-level statements' % len(top_stmts)) conflicts = pa.find_contradicts() top_stmts = remove_contradicts(top_stmts, conflicts) ac.dump_statements(top_stmts, 'pi_mtg_demo.pkl') return top_stmts