def test_build_with_no_parents(self):
kb = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb.get_query(['Work Hard'])
builder = CPTBuilder(data, kb.get_scale())
print(builder.build_with_no_parents())
kb1 = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb1.get_query(['Smart'])
builder1 = CPTBuilder(data, kb.get_scale())
print(builder1.build_with_no_parents())
def test_CPTBuilder(self):
kb = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb.get_query(['Success', 'Smart', 'Work Hard'])
builder = CPTBuilder(data, kb.get_scale())
print()
def resolve(self, kb):
print("EventRemovalResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(EventRemovalResolver, self).copy_all(resolved_kb, kb)
resolved_kb.clear_events_relations_and_groups()
bad_event_ids = set()
for evid, event in kb.get_events():
found_good_event_mention = False
for event_mention in event.event_mentions:
if event_mention.trigger is None or event_mention.trigger.lower() not in self.bad_trigger_words:
found_good_event_mention = True
break
if found_good_event_mention:
corrected_event = self.correct_event_type(event)
if corrected_event is not None:
resolved_kb.add_event(event)
else:
bad_event_ids.add(evid)
for relid, relation in kb.get_relations():
if relation.left_argument_id not in bad_event_ids and relation.right_argument_id not in bad_event_ids:
resolved_kb.add_relation(relation)
return resolved_kb
def resolve(self, kb, parameter):
print("Resolving KB with parameter: " + parameter)
resolved_kb = KnowledgeBase()
super(StructuredResolver, self).copy_all(resolved_kb, kb)
# Load document to ID mappings
# TODO: Need to determine what to do about zips
# Structured JSON data contains the full name for the zip,
# i.e. the "rel_path":
# "World_Bank_Data\API_BEN_DS2_en_csv_v2.zip\API_BEN_DS2_en_csv_v2.csv"
# but the current ID data file only goes to the zip filename
# TODO: is this now outdated based on file ids in json to be processed?
with io.open(os.path.dirname(os.path.realpath(__file__)) +
"/../data_files/structured_data_unique_ids.txt",
mode="r",
encoding="utf-8") as f:
for line in f:
(data_id, f_name) = line.split('|')
data_id = data_id.strip()
f_name = f_name.strip()
f_name = ascii_me(ntpath.basename(f_name))
self.id_manager.structured_document_name_to_id[
f_name] = data_id
resolved_kb.structured_documents = self._resolve_structured_inputs(kb)
resolved_kb.structured_relationships = \
self._resolve_structured_relationships(kb)
return resolved_kb
def __init__(self):
self.kb = KnowledgeBase()
self.lock = threading.Lock()
self.location = Location(self.kb, self.lock)
self.motion = MotionSubsystem(self.kb, self.lock)
def resolve(self, kb):
print("BadRelationResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(BadRelationResolver, self).copy_all(resolved_kb, kb)
resolved_kb.clear_relations_and_relation_groups()
for relid, relation in kb.get_relations():
# Always let event-event relations through
if relation.argument_pair_type == "event-event":
resolved_kb.add_relation(relation)
continue
# Entity-level arguments are the same, skip
if relation.left_argument_id == relation.right_argument_id:
#print "Skipping relation (1): " + relation.id
continue
# Cross-document entities will end up the same if the
# args have the same cameo_country_code, so skip
left_entity = kb.entid_to_kb_entity[relation.left_argument_id]
right_entity = kb.entid_to_kb_entity[relation.right_argument_id]
if (left_entity.properties.get("cameo_country_code") is not None and
right_entity.properties.get("cameo_country_code") is not None and
left_entity.properties.get("cameo_country_code") == right_entity.properties.get("cameo_country_code")):
#print "Skipping relation (2): " + relation.id
continue
resolved_kb.add_relation(relation)
return resolved_kb
def resolve(self, kb):
print("RedundantEventResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(RedundantEventResolver, self).copy_all(resolved_kb, kb)
resolved_kb.clear_events_relations_and_groups()
# Organize events by sentence
sentence_to_event_mention_list = dict()
event_mention_to_event = dict()
for evid, event in kb.get_events():
for event_mention in event.event_mentions:
event_mention_to_event[event_mention] = event
if event_mention.model == "ACCENT":
continue
sentence = event_mention.sentence
if sentence not in sentence_to_event_mention_list:
sentence_to_event_mention_list[sentence] = []
sentence_to_event_mention_list[sentence].append(event_mention)
event_mentions_to_remove = set()
for sentence, event_mention_list in sentence_to_event_mention_list.items(
):
# Looking at event mentions for a particular sentence
for em1 in event_mention_list:
for em2 in event_mention_list:
if em1 == em2:
continue
if em1.is_similar_and_better_than(em2):
#print "Throwing out: " + em2.id + " because it is worse than " + em1.id
event_mentions_to_remove.add(em2)
bad_event_ids = set()
for evid, event in kb.get_events():
found_good_event_mention = False
for event_mention in event.event_mentions:
if not event_mention in event_mentions_to_remove:
found_good_event_mention = True
break
if found_good_event_mention:
resolved_kb.add_event(event)
else:
bad_event_ids.add(evid)
for relid, relation in kb.get_relations():
if relation.left_argument_id not in bad_event_ids and relation.right_argument_id not in bad_event_ids:
resolved_kb.add_relation(relation)
# add event group to event mapping:
for evgid, event_group in kb.get_event_groups():
ev_group = KBEventGroup(event_group.id)
for event in event_group.members:
if event.id not in bad_event_ids:
ev_group.members.append(event)
if len(ev_group.members) > 0:
resolved_kb.add_event_group(ev_group)
return resolved_kb
def __init__(self):
self.kb = KnowledgeBase()
self.location = BarcodeServer(self.location_callback)
self.flight_path = FlightPath()
self.motion = Motion(self.kb)
self.node_a = NodeAServer()
def __init__(self):
self.kb = KnowledgeBase()
self.data = NodeBData()
self.packet = Packet('B')
self.radio = RadioAPI()
self.tx_packet_number = 1
self.rx_packet_list = []
def resolve(self, kb):
print("RemovalByTypeResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(RemovalByTypeResolver, self).copy_all(resolved_kb, kb)
resolved_kb.clear_events_relations_and_groups()
bad_event_ids = set()
for evid, event in kb.get_events():
for event_mention in event.event_mentions:
rule_list = self.removal_info.get(event_mention.event_type)
if not rule_list:
break
for rule in rule_list:
if rule.matches_event_mention(event_mention, kb):
bad_event_ids.add(evid)
break
bad_relation_ids = set()
for relid, relation in kb.get_relations():
for relation_mention in relation.relation_mentions:
rule_list = self.removal_info.get(relation.relation_type)
if not rule_list:
break
for rule in rule_list:
if rule.matches_relation_mention(relation_mention, kb):
bad_relation_ids.add(relid)
break
# Add in non-bad events to resolved KB
for evid, event in kb.get_events():
if evid not in bad_event_ids:
resolved_kb.add_event(event)
#else:
# print "Removing: " + evid
# Add in non-bad relations that didn't have an event removed
for relid, relation in kb.get_relations():
if (relid not in bad_relation_ids
and relation.left_argument_id not in bad_event_ids
and relation.right_argument_id not in bad_event_ids):
resolved_kb.add_relation(relation)
#else:
# print "Removing: " + relid
# add event group to event mapping:
for evgid, event_group in kb.get_event_groups():
ev_group = KBEventGroup(event_group.id)
for event in event_group.members:
if event.id not in bad_event_ids:
ev_group.members.append(event)
if len(ev_group.members) > 0:
resolved_kb.add_event_group(ev_group)
return resolved_kb
def __init__(self):
self.kb = KnowledgeBase()
self.radio = RadioAPI()
self.packet = Packet('B')
self.data = []
for i in range(50):
self.data.append(0xff)
self.tx_packet_number = 1
def test_apply_combination_filter(self):
kb = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb.get_query(['Success', 'Smart', 'Work Hard'])
builder = CPTBuilder(data, kb.get_scale())
print(builder._apply_combination_filter(['T', 'T']))
print(builder._apply_combination_filter(['F', 'T']))
print(builder._apply_combination_filter(
['A', 'B'])) # Combination is not apart of scale
def load_to_knowledge_base(filename, sqlite_filename="knowledge_base.sql"):
exceptions = load_exceptions(filename)
templates = [
ExceptionTemplate(exception.exception, [], exception.message)
for exception in exceptions
]
base = KnowledgeBase(sqlite_filename)
for template in templates:
if "{*}" in template.template:
base.insert_exception_template(template)
def resolve(self, kb):
print("RelevantKBEntityMentionResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(RelevantKBEntityMentionResolver, self).copy_all(resolved_kb, kb)
self.kb_entity_mention_to_kb_entity = dict()
self.kb_entity_to_kb_entity_group = dict()
self.actor_id_to_kb_entity_group = dict()
self.marked_kb_elements = set()
for entgroupid, kb_entity_group in resolved_kb.entgroupid_to_kb_entity_group.items():
kb_entity_group.is_referred_in_kb = False
for kb_entity in kb_entity_group.members:
kb_entity.is_referred_in_kb = False
self.kb_entity_to_kb_entity_group[kb_entity] = kb_entity_group
for kb_entity_mention in kb_entity.mentions:
kb_entity_mention.is_referred_in_kb = False
self.kb_entity_mention_to_kb_entity[kb_entity_mention] = kb_entity
if kb_entity_group.actor_id is not None:
self.actor_id_to_kb_entity_group[kb_entity_group.actor_id] = kb_entity_group
start_searching_points = set()
for evt_id,kb_event in resolved_kb.evid_to_kb_event.items():
for kb_event_mention in kb_event.event_mentions:
for arg_role,args in kb_event_mention.arguments.items():
for arg,score in args:
if isinstance(arg, KBMention):
self.mark_from_bottom_to_top(arg)
start_searching_points.add(self.kb_entity_mention_to_kb_entity[arg])
OUT_DEGREE = 2
while OUT_DEGREE > 0:
alternative_searching_points = set()
for relid, kb_relation in resolved_kb.relid_to_kb_relation.items():
if kb_relation.argument_pair_type == "entity-entity":
left_entity = resolved_kb.entid_to_kb_entity[kb_relation.left_argument_id]
right_entity = resolved_kb.entid_to_kb_entity[kb_relation.right_argument_id]
if left_entity in start_searching_points:
if right_entity not in start_searching_points and right_entity.is_referred_in_kb is False:
alternative_searching_points.add(right_entity)
self.mark_from_bottom_to_top(right_entity)
if right_entity in start_searching_points:
if left_entity not in start_searching_points and left_entity.is_referred_in_kb is False:
alternative_searching_points.add(left_entity)
self.mark_from_bottom_to_top(left_entity)
OUT_DEGREE -= 1
start_searching_points = alternative_searching_points
return resolved_kb
def generate_to_knowledge_base(filename,
sqlite_filename="knowledge_base.sql",
scorer=fuzz.ratio):
templates = generate_templates(filename, scorer)
base = KnowledgeBase(sqlite_filename)
for template in templates:
if not template.generated:
continue
base.insert_exception_template(template)
def test_get_class_data(self):
kb = KnowledgeBase(None, '..\\..\\test_data\\Test_Data_AB.csv')
kb.add_data('..\\..\\test_data\\Test_Data_C.csv')
print(kb.get_data())
print()
print(kb.get_class_data(['A', 'C']))
print()
print(kb.get_class_data(['B']))
print(kb.get_class_data(['Z']))
print(kb.get_class_data(['A', 'B', 'Z']))
def resolve(self, kb):
print("EntityAddPropertyResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(EntityAddPropertyResolver, self).copy_all(resolved_kb, kb)
for entid, kb_entity in resolved_kb.entid_to_kb_entity.items():
if kb_entity.canonical_name is None:
self.pick_canonical_name(kb_entity)
return resolved_kb
def resolve(self, kb):
print("AddGenericEventTypeIfOnlyCausalFactorTypeAvailableResolver RESOLVE")
generic_event_uri = "http://ontology.causeex.com/ontology/odps/Event#Event"
resolved_kb = KnowledgeBase()
super(AddGenericEventTypeIfOnlyCausalFactorTypeAvailableResolver, self).copy_all(resolved_kb, kb)
for kb_event_id, kb_event in resolved_kb.evid_to_kb_event.items():
for kb_event_mention in kb_event.event_mentions:
if len(kb_event_mention.external_ontology_sources) < 1 and len(kb_event_mention.causal_factors) > 0:
highest_cf_score = max(cf.relevance for cf in kb_event_mention.causal_factors)
kb_event_mention.external_ontology_sources.append([generic_event_uri,highest_cf_score])
return resolved_kb
def __init__(self):
self.kb = KnowledgeBase()
self.lock = Lock()
self.data = NodeAData()
self.packet = Packet('A')
self.radio = RadioAPI()
self.tx_packet_number = 1
self.ack_packet_number = 0
self.goodput = 0
def resolve(self, kb):
print("RedundantRelationResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(RedundantRelationResolver, self).copy_all(resolved_kb, kb)
resolved_kb.clear_relations_and_relation_groups()
# Organize old relation mentions by sentence
sentences_to_relation_mention_list = dict()
relation_mention_to_relation = dict()
for relid, relation in kb.get_relations():
if relation.argument_pair_type != "event-event":
continue
for relation_mention in relation.relation_mentions:
relation_mention_to_relation[relation_mention] = relation
left_sentence = relation_mention.left_mention.sentence
right_sentence = relation_mention.right_mention.sentence
k = (
left_sentence,
right_sentence,
)
if k not in sentences_to_relation_mention_list:
sentences_to_relation_mention_list[k] = []
sentences_to_relation_mention_list[k].append(relation_mention)
relation_mentions_to_remove = set()
for sentence_pair, relation_mention_list in sentences_to_relation_mention_list.items(
):
# Essentially looking at the relations mentions for a particular sentence here
for rm1 in relation_mention_list:
for rm2 in relation_mention_list:
if rm1 == rm2:
continue
r1 = relation_mention_to_relation[rm1]
r2 = relation_mention_to_relation[rm2]
if rm1.is_similar_and_better_than(rm2, r1.relation_type,
r2.relation_type):
#print("Throwing out: " + rm2.id + " because it's worse than: " + rm1.id)
relation_mentions_to_remove.add(rm2)
for relid, relation in kb.get_relations():
found_good_relation_mention = False
for relation_mention in relation.relation_mentions:
if relation_mention not in relation_mentions_to_remove:
found_good_relation_mention = True
break
if found_good_relation_mention:
resolved_kb.add_relation(relation)
return resolved_kb
def main():
'''User-Inputted Recipie Title'''
#url = generateURL()
'''Hard Coded URLs'''
# url = "http://allrecipes.com/recipe/brown-rice-and-quinoa-sushi-rolls/"
# url = "http://allrecipes.com/recipe/Boilermaker-Tailgate-Chili/"
#url = "http://allrecipes.com/recipe/jerk-chicken/"
#link = urllib.urlopen(url)
#page = link.read()
'''Local Cached Webpages'''
# url = "../data/Burger"
# url = "../data/Cake"
url = "../data/Stir-Fry"
f = open(url + ".html")
page = f.read()
if len(sys.argv) < 2:
recipe = parse_recipe(page)
recipe = unabridgeMeasure(recipe)
prettyPrintRecipe(recipe)
save_output(url, recipe)
elif len(sys.argv) == 2:
case_num = int(sys.argv[1])
if case_num < 1 or case_num > 8:
print('You can only pick from [1-8]')
return
recipe = parse_recipe(page)
recipe = unabridgeMeasure(recipe)
kb = KnowledgeBase()
if case_num == 1:
tf_recipe = kb.transform_cuisine("italian", recipe)
elif case_num == 2:
tf_recipe = kb.transform_cuisine("chinese", recipe)
elif case_num == 3:
tf_recipe = kb.transform_diet("vegetarian", recipe)
elif case_num == 4:
tf_recipe = kb.transform_diet("pescatarian", recipe)
elif case_num == 5:
tf_recipe = kb.transform_healthy("low-fat", recipe)
elif case_num == 6:
tf_recipe = kb.transform_healthy("low-sodium", recipe)
elif case_num == 7:
tf_recipe = transformQty(2, recipe)
elif case_num == 8:
tf_recipe = transformQty(3, recipe)
prettyPrintRecipe(tf_recipe)
save_output(url, tf_recipe)
else:
print(
'Too many arguments. You can either either just call main.py to see the recipe or pass in a single integer to select a transformation.'
)
def __init__(self, agent):
self._agent = agent
self._cooperation = {}
self._knowledge_base = KnowledgeBase()
self._current_goal = self._knowledge_base.goals['find_trash']
# time awareness
self._time_trash_found = self.time
self._agent_trash_count_history = self._agent.trash_count
# queue to maintain last 10 positions
self._agent_position_history = deque([(0, 0)], 10)
# domain awareness
self._time_domain_strategy_applied = self.time
def __init__(self):
self.kb = KnowledgeBase()
self.lock = threading.Lock()
self.location = Location(self.kb, self.lock)
self.motion = SimpleMotion(self.kb, self.lock)
self.rf = RadioSubsystem(self.kb, self.lock, self.radio_data_callback)
self.fsm_state = 'at_beginning'
self.sent_packet = np.array([])
self.ack_packet = np.array([])
self.goodput = np.array([])
def test_calculate_probability(self):
kb = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb.get_query(['Success', 'Smart', 'Work Hard'])
builder = CPTBuilder(data, kb.get_scale())
fd = list(builder._apply_combination_filter(['T', 'T']))
self.assertTrue(.8 == builder._calculate_probability(fd, 'T'))
fd = list(builder._apply_combination_filter(['T', 'C']))
self.assertTrue(0 == builder._calculate_probability(fd, 'T'))
kb1 = KnowledgeBase('..\\..\\..\\test_data\\doug_example.csv',
'..\\..\\..\\test_data\\doug_example.csv')
data = kb1.get_query(['Work Hard'])
builder1 = CPTBuilder(data, kb.get_scale())
self.assertTrue(
(11 /
24) == builder1._calculate_probability(list(data.values), 'T'))
self.assertTrue(
(13 /
24) == builder1._calculate_probability(list(data.values), 'F'))
def __init__(self):
self.lock = Lock()
self.kb = KnowledgeBase()
self.location = Location(self.kb, self.lock)
self.motion = MotionSubsystem(self.kb, self.lock, self.motion_callback)
self.rf = RadioSubsystem(self.kb, self.lock, self.radio_data_callback)
self.fsm_state = 'at_beginning'
self.sent_packet = np.array([])
self.ack_packet = np.array([])
self.goodput = np.array([])
self.color_values = np.array([])
self.arrived = False
def resolve(self, kb):
resolved_kb = KnowledgeBase()
super(DropNegativePolarityCausalAssertionResolver,
self).copy_all(resolved_kb, kb)
new_relid_to_kb_relation = dict()
for relid, kb_relation in resolved_kb.relid_to_kb_relation.items():
if kb_relation.argument_pair_type != "event-event":
new_relid_to_kb_relation[relid] = kb_relation
elif kb_relation.polarity != "Negative":
new_relid_to_kb_relation[relid] = kb_relation
else:
src_kb_event = resolved_kb.evid_to_kb_event[
kb_relation.left_argument_id]
tar_kb_event = resolved_kb.evid_to_kb_event[
kb_relation.right_argument_id]
logger.debug("NCA Dropping {}".format(
get_marked_up_string_for_event_event_relation(
kb_relation, src_kb_event, tar_kb_event)))
resolved_kb.relid_to_kb_relation = new_relid_to_kb_relation
return resolved_kb
def __init__(self, inputfile=None):
# For reading/writing input/output
if inputfile == None:
self.inputFileName = self.getInputFileName()
else:
self.inputFileName = inputfile
self.inputFile = open(self.inputFileName, 'r')
self.outputFileName = "output.txt"
self.outputFile = open(self.outputFileName, 'w')
# Num queries
self.numQueries = 0
# Num statements for the kb
self.numStatements = 0
# List of queries. Stored as predicate objects.
self.queryList = []
# The knowledge base for this set of queries
self.kb = KnowledgeBase()
def resolve(self, kb, awake_db):
print("ExternalURIResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(ExternalURIResolver, self).copy_all(resolved_kb, kb)
if awake_db == "NA":
return resolved_kb
kb_entity_to_entity_group = dict()
for entgroupid, kb_entity_group in resolved_kb.get_entity_groups():
for kb_entity in kb_entity_group.members:
kb_entity_to_entity_group[kb_entity] = kb_entity_group
AwakeDB.initialize_awake_db(awake_db)
for entid, kb_entity in resolved_kb.entid_to_kb_entity.items():
kb_entity_group = kb_entity_to_entity_group[kb_entity]
source_string = AwakeDB.get_source_string(kb_entity_group.actor_id)
if source_string is not None and source_string.find(
"dbpedia.org") != -1:
formatted_string = source_string.strip()
if source_string.startswith("<"):
source_string = source_string[1:]
if source_string.endswith(">"):
source_string = source_string[0:-1]
source_string = source_string.replace("dbpedia.org/resource",
"en.wikipedia.org/wiki",
1)
kb_entity.properties["external_uri"] = source_string
# For countries, add geoname_id to properties
if (kb_entity_group.actor_id is not None
and "external_uri" not in kb_entity.properties
and "geonameid" not in kb_entity.properties):
geonameid = AwakeDB.get_geonameid_from_actorid(
kb_entity_group.actor_id)
if geonameid is not None and len(str(geonameid).strip()) != 0:
kb_entity.properties["geonameid"] = str(geonameid)
return resolved_kb
def resolve(self, kb):
print("EventPolarityResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(EventPolarityResolver, self).copy_all(resolved_kb, kb)
for evid, kb_event in resolved_kb.get_events():
for kb_event_mention in kb_event.event_mentions:
event_mention_text = kb_event_mention.trigger
should_reverse_polarity = False
tokens_in_mention = event_mention_text.split(" ")
for token in tokens_in_mention:
if token in negative_word_set and len(
tokens_in_mention) > 1:
should_reverse_polarity = True
if should_reverse_polarity is True:
for event_type, grounding in kb_event_mention.external_ontology_sources:
for negative_type_keyword in negative_type_set:
if negative_type_keyword in event_type:
should_reverse_polarity = False
if should_reverse_polarity is True:
kb_event_mention.properties["polarity"] = "Negative"
return resolved_kb
def resolve(self, kb):
print("EntityMentionNumericResolver RESOLVE")
resolved_kb = KnowledgeBase()
super(EntityMentionNumericResolver, self).copy_all(resolved_kb, kb)
for evtid, kb_event in resolved_kb.evid_to_kb_event.items():
for kb_event_mention in kb_event.event_mentions:
for arg_role, args in kb_event_mention.arguments.items():
if arg_role in {
'has_actor', 'has_active_actor',
'has_affected_actor'
}:
for arg in args:
if isinstance(arg, KBMention):
kb_mention = arg
original_sentence_str = kb_mention.sentence.text
mention_str = kb_mention.mention_text
possible_numeric_dict = extract_numeric_for_mention(
mention_str, mention_str)
possible_numeric_dict_extra = extract_numeric_for_mention(
original_sentence_str, mention_str)
if len(possible_numeric_dict.keys()) > 0:
for key in possible_numeric_dict_extra.keys(
):
if key not in {
'val', 'min_val', 'max_val'
}:
possible_numeric_dict[
key] = possible_numeric_dict_extra[
key]
kb_mention.properties[
'numeric'] = possible_numeric_dict
return resolved_kb