def test_create_event(self): g, system = get_initialized_graph_and_system() # make event with type election election = aifutils.make_event(g, prefix + "event/1", system) aifutils.mark_type(g, prefix + "types/1", election, "Q40231", system, 1.0) # make entity with type person putin = aifutils.make_entity(g, prefix + "entity/putin", system) aifutils.mark_type(g, prefix + "types/2", putin, "Q5", system, 1.0) # make entity with type political region russia = aifutils.make_entity(g, prefix + "enitty/russia", system) aifutils.mark_type(g, prefix + "types/3", russia, "Q1048835", system, 1.0) # add putin as cadidate for the election aifutils.mark_as_argument(g, election, "A1_ppt_theme_candidate", putin, system, .785, prefix + "arg/candidate") # add russia as location for election aifutils.mark_as_argument(g, election, "AM_loc__location", russia, system, .589, prefix + "arg/location") self.new_file(g, "test_create_event.ttl") self.dump_graph(g, "Example of event and entity")
def test_create_an_event_argument_add_invalid_attribute(self): #g = get_initialized_graph() g = aifutils.make_graph() # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, "http://www.test.edu/testSystem") # we make a resource for the event itself event = aifutils.make_event(g, "http://www.test.edu/events/1", system) # mark the event as a Personnel.Elect event; type is encoded separately so we can express # uncertainty about type aifutils.mark_type(g, "http://www.test.edu/assertions/5", event, ldc_ontology.Personnel_Elect, system, 1.0) # create the two entities involved in the event electee = aifutils.make_entity(g, "http://www.test.edu/entities/1", system) aifutils.mark_type(g, "http://www.test.edu/assertions/6", electee, ldc_ontology.PER, system, 1.0) election_country = aifutils.make_entity( g, "http://www.test.edu/entities/2", system) aifutils.mark_type(g, "http://www.test.edu/assertions/7", election_country, ldc_ontology.GPE, system, 1.0) # link those entities to the event argument1 = aifutils.mark_as_argument( g, event, ldc_ontology.Personnel_Elect_Candidate, electee, system, 0.785) argument2 = aifutils.mark_as_argument( g, event, ldc_ontology.Personnel_Elect_Place, election_country, system, 0.589) aifutils.mark_attribute(g, argument1, interchange_ontology.Irrealis) aifutils.mark_attribute(g, argument1, interchange_ontology.Generic) aifutils.mark_attribute( g, argument2, interchange_ontology.VideoJustificationChannelPicture) aifutils.mark_attribute( g, argument2, interchange_ontology.VideoJustificationChannelSound) self.new_file( g, "test_create_an_event_argument_add_invalid_attribute.ttl") self.dump_graph( g, "Invalid: Semantic Attribute for Event Argument, must be aida:Negated, aida:Hedged" )
def test_create_a_relation_argument_add_attribute(self): #g = get_initialized_graph() g = aifutils.make_graph() # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, "http://www.test.edu/testSystem") bob = aifutils.make_entity(g, "http://www.test.edu/entites/person/Bob", system) maryland = aifutils.make_entity( g, "http://www.test.edu/entites/place/Maryland", system) aifutils.mark_type(g, "http://www.test.edu/assertions/bobIsAPerson", bob, ldc_ontology.PER, system, 1.0) aifutils.mark_type( g, "http://www.test.edu/assertions/marylandIsALocation", maryland, ldc_ontology.LOC_Position_Region, system, 1.0) # we make a resource for the event itself relationBobLiveInMD = aifutils.make_relation( g, "http://www.test.edu/relationss/bobLivesInMaryland", system) argument1 = aifutils.mark_as_argument( g, relationBobLiveInMD, ldc_ontology.Physical_Resident_Resident, bob, system, 1) aifutils.mark_attribute(g, argument1, interchange_ontology.Generic) self.new_file(g, "test_create_a_relation_argument_add_attribute.ttl") self.dump_graph( g, "Invalid: Relation Argument cannot have aida:Attribute")
def test_create_seedling_event_with_event_argument_uri(self): g = aifutils.make_graph() # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, "http://www.test.edu/testSystem") # we make a resource for the event itself event = aifutils.make_event(g, "http://www.test.edu/events/1", system) # mark the event as a Personnel.Elect event; type is encoded separately so we can express # uncertainty about type event_type_string = "Personnel.Elect" aifutils.mark_type(g, "http://www.test.edu/assertions/5", event, SEEDLING_TYPES_NIST[event_type_string], system, 1.0) # create the two entities involved in the event electee = aifutils.make_entity(g, "http://www.test.edu/entities/1", system) aifutils.mark_type(g, "http://www.test.edu/assertions/7", electee, SEEDLING_TYPES_NIST.Person, system, 1.0) election_country = aifutils.make_entity( g, "http://www.test.edu/entities/2", system) aifutils.mark_type(g, "http://www.test.edu/assertions/7", election_country, SEEDLING_TYPES_NIST.GeopoliticalEntity, system, 1.0) # link those entities to the event aifutils.mark_as_argument( g, event, SEEDLING_TYPES_NIST[event_type_string] + "_Elect", electee, system, .785, "http://www.test.edu/eventArgument/1") aifutils.mark_as_argument( g, event, SEEDLING_TYPES_NIST[event_type_string] + "_Place", election_country, system, .589, "http://www.test.edu/eventArgument/2") self.new_file( g, "test_create_seedling_event_with_event_argument_uri.ttl") self.dump_graph(g, "Example of seedling event with event assertion URI")
def test_create_an_event(self): g = aifutils.make_graph() g.bind('ldcOnt', SEEDLING_TYPES_NIST.uri) # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, "http://www.test.edu/testSystem") # we make a resource for the event itself event = aifutils.make_event(g, "http://www.test.edu/events/1", system) # mark the event as a Personnel.Elect event; type is encoded separately so we can express # uncertainty about type aifutils.mark_type(g, "http://www.test.edu/assertions/5", event, SEEDLING_TYPES_NIST['Personnel.Elect'], system, 1.0) # create the two entities involved in the event electee = aifutils.make_entity(g, "http://www.test.edu/entities/1", system) aifutils.mark_type(g, "http://www.test.edu/assertions/6", electee, SEEDLING_TYPES_NIST.Person, system, 1.0) election_country = aifutils.make_entity( g, "http://www.test.edu/entities/2", system) aifutils.mark_type(g, "http://www.test.edu/assertions/7", election_country, SEEDLING_TYPES_NIST.GeopoliticalEntity, system, 1.0) # link those entities to the event arg = URIRef(SEEDLING_TYPES_NIST['Personnel.Elect'] + "_Elect") aifutils.mark_as_argument(g, event, arg, electee, system, 0.785) arg2 = URIRef(SEEDLING_TYPES_NIST['Personnel.Elect'] + "_Place") aifutils.mark_as_argument(g, event, arg2, election_country, system, 0.589) self.new_file(g, "test_create_an_event.ttl") self.dump_graph(g, "Example of creating an event")
def add_event(self): # add an event event_resource = aifutils.make_event(self.g, self.get_event_uri(), self.system) # add the type type_resource = self.EVENT_TYPES[ random.randint(0, len(self.EVENT_TYPES)) - 1] type_assertion = aifutils.mark_type(self.g, self.get_assertion_uri(), event_resource, type_resource, self.system, 1.0) self.add_justification_and_private_data(type_assertion) # make two arguments for i in range(2): arg = URIRef(type_resource + self.get_random_suffix()) argument = aifutils.mark_as_argument(self.g, event_resource, arg, self.get_random_entity(), self.system, 0.785, self.get_assertion_uri()) self.add_justification_and_private_data(argument)
def test_simple_hypothesis_with_importance_with_cluster(self): g = aifutils.make_graph() g.bind('ldcOnt', SEEDLING_TYPES_NIST.uri) # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, 'http://www.test.edu/testSystem') # buk document entity buk = aifutils.make_entity(g, "E779954.00005", system) buk_is_weapon = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-1", buk, SEEDLING_TYPES_NIST.Weapon, system, 1.0) # buk cross-document-entity buk_kb_entity = aifutils.make_entity( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#E0084", system) buk_kb_is_weapon = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-2", buk_kb_entity, SEEDLING_TYPES_NIST.Weapon, system, 1.0) # russia document entity russia = aifutils.make_entity( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#E779954.00004", system) russia_is_gpe = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-3", russia, SEEDLING_TYPES_NIST.GeopoliticalEntity, system, 1.0) # cluster buk buk_cluster = aifutils.make_cluster_with_prototype( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#cluster-1", buk_kb_entity, system) buk_is_clustered = aifutils.mark_as_possible_cluster_member( g, buk, buk_cluster, .9, system) # add importance to the cluster - test negative importance aifutils.mark_importance(g, buk_cluster, -70.234) # Russia owns buk relation buk_is_russian = aifutils.make_relation( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#R779959.00004", system) aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-4", buk_is_russian, SEEDLING_TYPES_NIST['GeneralAffiliation.APORA'], system, 1.0) buk_argument = aifutils.mark_as_argument( g, buk_is_russian, SEEDLING_TYPES_NIST['GeneralAffiliation.APORA_Affiliate'], buk, system, 1.0) russia_argument = aifutils.mark_as_argument( g, buk_is_russian, SEEDLING_TYPES_NIST['GeneralAffiliation.APORA_Affiliation'], russia, system, 1.0) # add importance to the statements aifutils.mark_importance(g, buk_argument, 100.0) # add large importance aifutils.mark_importance(g, russia_argument, 9.999999e6) # Russia owns buk hypothesis buk_is_russian_hypothesis = aifutils.make_hypothesis( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#hypothesis-1", [ buk, buk_is_weapon, buk_is_clustered, buk_is_russian, buk_argument, russia_argument ], system) # test highest possible importance value aifutils.mark_importance(g, buk_is_russian_hypothesis, sys.float_info.max) self.new_file(g, "test_simple_hypothesis_with_importance_cluster.ttl") self.dump_graph(g, "Simple hypothesis with importance with cluster")
def test_create_compound_justification(self): g = aifutils.make_graph() system = aifutils.make_system_with_uri(g, "http://www.test.edu/system") event = aifutils.make_event( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#V779961.00010", system) event_type_assertion = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-1", event, SEEDLING_TYPES_NIST['Personnel.Elect'], system, 1.0) # create the two entities involved in the event putin = aifutils.make_entity( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#E781167.00398", system) person_type_assertion = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-2", putin, SEEDLING_TYPES_NIST.Person, system, 1.0) russia = aifutils.make_entity( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#E779954.00004", system) gpe_type_assertion = aifutils.mark_type( g, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-3", russia, SEEDLING_TYPES_NIST.GeopoliticalEntity, system, 1.0) # link those entities to the event electee_argument = aifutils.mark_as_argument( g, event, SEEDLING_TYPES_NIST['Personnel.Elect_Elect'], putin, system, 0.785, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-4" ) place_argument = aifutils.mark_as_argument( g, event, SEEDLING_TYPES_NIST['Personnel.Elect_Place'], russia, system, 0.589, "https://tac.nist.gov/tracks/SM-KBP/2019/LdcAnnotations#assertion-5" ) # the justification provides the evidence for our claim about the entity's type # we attach this justification to both the type assertion and the entity object itself, since it provides # evidence both for the entity's existence and its type. # in TA1 -> TA2 communications, we attach confidences at the level of justifications text_justification = aifutils.make_text_justification( g, "NYT_ENG_20181231", 42, 143, system, 0.973) aifutils.mark_justification(g, person_type_assertion, text_justification) aifutils.mark_justification(g, putin, text_justification) aifutils.add_source_document_to_justification( g, text_justification, "NYT_PARENT_ENG_20181231_03") bb1 = Bounding_Box((123, 45), (167, 98)) # let's suppose we also have evidence from an image image_justification = aifutils.make_image_justification( g, "NYT_ENG_20181231_03", bb1, system, 0.123) bb2 = Bounding_Box((234, 56), (345, 101)) # and also a video where the entity appears in a keyframe keyframe_video_justification = aifutils.make_keyframe_video_justification( g, "NYT_ENG_20181231_03", "keyframe ID", bb2, system, .0234) #and also a video where the entity does not appear in a keyframe shot_video_justification = aifutils.make_shot_video_justification( g, "SOME_VIDEO", "some shot ID", system, 0.487) # and even audio! audio_justification = aifutils.make_audio_justification( g, "NYT_ENG_201181231", 4.566, 9.876, system, 0.789) # combine all justifications into single justifiedBy triple with new confidence aifutils.mark_compound_justification(g, [electee_argument], [ text_justification, image_justification, keyframe_video_justification, shot_video_justification, audio_justification ], system, .321) aifutils.mark_compound_justification( g, [place_argument], [text_justification, image_justification], system, 0.543) self.new_file(g, "test_create_compound_justification.ttl") self.dump_graph(g, "Example of compound justification")
def test_use_subgraph_confidences_to_show_mutually_exclusive_linked_event_argument_options( self): g = aifutils.make_graph() g.bind('ldcOnt', SEEDLING_TYPES_NIST.uri) # every AIF needs an object for the system responsible for creating it system = aifutils.make_system_with_uri( g, "http://www.test.edu/testSystem") # we make a resource for the event itself event = aifutils.make_event(g, "http://www.test.edu/events/1", system) # mark the event as a Personnel.Elect event; type is encoded separately so we can express uncertainty about type aifutils.mark_type(g, "http://www.test.edu/assertions/5", event, SEEDLING_TYPES_NIST['Conflict.Attack'], system, 1.0) # create the two entities involved in the event bob = aifutils.make_entity(g, "http://www.test.edu/entities/1", system) aifutils.mark_type(g, "http://www.test.edu/assertions/6", bob, SEEDLING_TYPES_NIST.Person, system, 1.0) fred = aifutils.make_entity(g, "http://www.test.edu/entities/2", system) aifutils.mark_type(g, "http://www.test.edu/assertions/7", fred, SEEDLING_TYPES_NIST.Person, system, 1.0) # we link all possible argument fillers to the event bob_hit_fred_assertions = [ aifutils.mark_as_argument( g, event, URIRef(SEEDLING_TYPES_NIST['Conflict.Attack'] + "_Attacker"), bob, system, None), aifutils.mark_as_argument( g, event, URIRef(SEEDLING_TYPES_NIST['Conflict.Attack'] + "_Target"), fred, system, None) ] fred_hit_bob_assertions = [ aifutils.mark_as_argument( g, event, URIRef(SEEDLING_TYPES_NIST['Conflict.Attack'] + "_Attacker"), fred, system, None), aifutils.mark_as_argument( g, event, URIRef(SEEDLING_TYPES_NIST['Conflict.Attack'] + "_Target"), bob, system, None) ] # then we mark these as mutually exclusive # we also mark confidence 0.2 that neither of these are true aifutils.mark_as_mutually_exclusive( g, { tuple(bob_hit_fred_assertions): 0.6, tuple(fred_hit_bob_assertions): 0.2 }, system, 0.2) self.new_file( g, "test_use_subgraph_confidences_to_show_mutually_exclusive_linked_event_argument_options.ttl" ) self.dump_graph( g, "Example of subgraph confidences to show mutually exclusive linked event argument options" )
def transferAIF(parent): chi = child[parent] if parent in kb_dict.keys(): g = kb_dict[parent] else: g = aifutils.make_graph() entityDic_c = {} country_set_r = set() for img_id in chi: if img_id in flag_dict_s.keys(): country_set_r.add(flag_dict_s[img_id]) #=============== Flag =========== sys = aifutils.make_system_with_uri( g, "http://www.columbia.edu/AIDA/DVMM/Systems/Flag/Inception_v4") for key, value in index_category_f.items(): if key not in country_set_r: continue key = key.replace(' ', '_') name = "http://www.columbia.edu/AIDA/DVMM/Entities/Country/" + str( value) + '/' + key entity = aifutils.make_entity(g, name, sys) entityDic_c[key] = entity #print "parent "+ parent type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/Country/"\ +str(value)+'/'+key, entity, AIDA_PROGRAM_ONTOLOGY_G.term('GPE'), sys, 1) for img_id in chi: if img_id in flag_dict_s.keys(): #print "imgID "+img_id flag_label = [ '/m/07cmd', '/m/0dzct', '/m/03bt1vf', '/m/01g317', '/m/04yx4', '/m/01prls', '/m/07yv9', '/m/03120' ] key = img_id if key in OD_result.keys(): for n in range(len(OD_result[key])): eid = "http://www.columbia.edu/AIDA/DVMM/Entities/ObjectDetection/RUN00010/JPG/" + key + "/" + str( n) #print OD_result[key][n]['label'] if OD_result[key][n]['label'] in flag_label: #print OD_result[key][n]['label'] eid = "http://www.columbia.edu/AIDA/DVMM/Entities/ObjectDetection/RUN00010/JPG/" + key + "/" + str( n) relation_entity = aifutils.make_relation(g, "http://www.columbia.edu/AIDA/DVMM/Relaion/Flag/"+\ key+"/"+str(n), sys) #APORA = 'GeneralAffiliation.ArtifactPoliticalOrganizationReligiousAffiliation' score = 1 type_assertion = aifutils.mark_type(g, "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/Flag/"+key+"/"+str(n), \ relation_entity, AIDA_PROGRAM_ONTOLOGY_A['GeneralAffiliation.ArtifactPoliticalOrganizationReligiousAffiliation.NationalityCitizen'], \ sys, score) boxA = OD_result[key][n]['bbox'] bb2 = Bounding_Box((boxA[0], boxA[1]), (boxA[2], boxA[3])) score = 1 justif = aifutils.mark_image_justification(g, [relation_entity, \ type_assertion], key, bb2, sys, score) aifutils.add_source_document_to_justification( g, justif, parent) aifutils.mark_informative_justification( g, relation_entity, justif) score = 1 if eid in entity_dict.keys(): art_argument = aifutils.mark_as_argument(g, relation_entity, \ AIDA_PROGRAM_ONTOLOGY_A['GeneralAffiliation.ArtifactPoliticalOrganizationReligiousAffiliation.NationalityCitizen_Artifact'], \ entity_dict[eid], sys, score) score = 1 justif = aifutils.mark_image_justification(g, [relation_entity, \ art_argument], key, bb2, sys, score) aifutils.add_source_document_to_justification( g, justif, parent) aifutils.mark_informative_justification( g, relation_entity, justif) score = 1 nation_argument = aifutils.mark_as_argument(g, relation_entity, \ AIDA_PROGRAM_ONTOLOGY_A['GeneralAffiliation.ArtifactPoliticalOrganizationReligiousAffiliation.NationalityCitizen_Nationality'], \ entityDic_c[flag_dict_s[img_id]], \ sys, score) score = 1 justif = aifutils.mark_image_justification(g, [relation_entity, \ nation_argument], key, bb2, sys, score) aifutils.add_source_document_to_justification( g, justif, parent) aifutils.mark_informative_justification( g, relation_entity, justif) if OD_result[key][n]['label'] == '/m/03120': #print flag_dict_s[img_id] boxA = OD_result[key][n]['bbox'] bb2 = Bounding_Box((boxA[0], boxA[1]), (boxA[2], boxA[3])) type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/Country/RUN00010/JPG/"+\ str(key)+'/'+str(n), entityDic_c[flag_dict_s[img_id]], AIDA_PROGRAM_ONTOLOGY_G.term('GPE'), sys, 1) score = 1 justif = aifutils.mark_image_justification(g, [entityDic_c[flag_dict_s[img_id]], type_assertion], \ key, bb2, sys, score) aifutils.add_source_document_to_justification( g, justif, parent) aifutils.mark_informative_justification( g, entityDic_c[flag_dict_s[img_id]], justif) #break #=============== landmark ========== sys = aifutils.make_system_with_uri( g, "http://www.columbia.edu/AIDA/DVMM/Systems/Landmark/Delf") #building has_land = 0 landmark_name_set = set() for imageN in chi: if imageN in landmark_dict.keys(): name_lm = landmark_dict[imageN] if name_lm == '': continue #print name_lm if name_lm not in landmark_name_set: landmark_name_set.add(name_lm) name = "http://www.columbia.edu/AIDA/DVMM/Entities/Landmark/"+ \ str(landmark_id[name_lm])+"/"+name_lm entity = aifutils.make_entity(g, name, sys) entityDic_b[name_lm] = entity #print AIDA_PROGRAM_ONTOLOGY2.term('Person') score = 1 type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/Landmark/"+ \ str(landmark_id[name_lm])+"/"+name_lm \ , entity, AIDA_PROGRAM_ONTOLOGY_b.term('FAC'), sys, score) if 'Maidan' in name_lm: aifutils.link_to_external_kb(g, entity, "LDC2019E43:80000020", sys, 1) elif 'United_States_Capitol' in name_lm: aifutils.link_to_external_kb(g, entity, "LDC2019E43:4140827", sys, 1) else: aifutils.link_to_external_kb(g, entity, VIS_ONTOLOGY_b.term(name_lm), sys, 1) for imageN in chi: total_key.add(imageN) if imageN in landmark_dict.keys(): #print parent key = imageN name_lm = landmark_dict[imageN] if name_lm == '': continue #aifutils.mark_as_possible_cluster_member(g, \ # entity,clusterDic_b[0], 1, sys) # Need to change in future im = Image.open(input_img_path + imageN + '.jpg') width, height = im.size bb2 = Bounding_Box((0, 0), (width, height)) #l u r d type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/Landmark/RUN00010/JPG/"+\ str(key)+'/'+str(0), entityDic_b[name_lm], AIDA_PROGRAM_ONTOLOGY_b.term('FAC'), sys, 1) justif = aifutils.mark_image_justification( g, [entityDic_b[name_lm], type_assertion], key, bb2, sys, 1) aifutils.add_source_document_to_justification(g, justif, parent) aifutils.mark_informative_justification(g, entityDic_b[name_lm], justif) #=============face recognition============= sys = aifutils.make_system_with_uri( g, "http://www.columbia.edu/AIDA/DVMM/Systems/Face/FaceNet") NameDetected = set() NameDetected_score = {} nameCount = Counter() count = 0 c_num = 0 person_c_n = 0 #print comblineSet In = 0 #print bb featureDic = {} first = 1 entityList = [] arrayList = [] person_label = [ '/m/01g317', '/m/04yx4', '/m/03bt1vf', '/m/01bl7v', '/m/05r655', '/m/04hgtk', '/m/01bgsw' ] for x, y in result.items(): #print x data = x.split('/')[-1] if '._' in data: continue data2 = data.split('_') #print data2[0] #print data2[1][:-4] key = data2[0] i = data2[1][:-4] #print chi #print key if key not in chi: continue if y[0].replace(' ', '_') not in nameSet or nameDic[y[0].replace( ' ', '_')] not in p1Set or float(y[1]) < 0.04 or y[0].replace( ' ', '_') == 'Ban_Ki-moon': continue else: NameDetected.add(nameDic[y[0].replace(' ', '_')]) score = float(y[1]) + 0.5 NameDetected_score[nameDic[y[0].replace(' ', '_')]] = min(1, score) for x, y in result2.items(): #print x data = x.split('/')[-2] if '._' in data: continue data = x.split('/')[-1] data2 = data.split('_') key = x.split('/')[-2] if videoDic[key] not in chi: continue if y[0].replace(' ', '_') not in nameSet or nameDic[y[0].replace( ' ', '_')] not in p1Set or float(y[1]) < 0.04 or y[0].replace( ' ', '_') == 'Ban_Ki-moon': continue else: NameDetected.add(nameDic[y[0].replace(' ', '_')]) score = float(y[1]) + 0.5 NameDetected_score[nameDic[y[0].replace(' ', '_')]] = min(1, score) for key, value in index_category.items(): #key name. value is number if key not in NameDetected: continue key = key.replace(' ', '_') #keu = entityDic name = "http://www.columbia.edu/AIDA/DVMM/Entities/FaceID/" + str( value) + '/' + key entity = aifutils.make_entity(g, name, sys) entityDic[key] = entity #print AIDA_PROGRAM_ONTOLOGY2.term('Person') #score = NameDetected_score[key] type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/FaceID/"\ +str(value)+'/'+key, entity, AIDA_PROGRAM_ONTOLOGY2.term('PER'), sys, 1) new_key = key.lower().replace('_', ' ') if new_key in name2ID.keys(): aifutils.link_to_external_kb(g, entity, "LDC2019E43:" + name2ID[new_key], sys, 1) #print 'Lorelei' #print parent else: a = 0 #print 'dbpedia' #print parent aifutils.link_to_external_kb(g, entity, VIS_ONTOLOGY2.term(key), sys, 1) for x, y in result.items(): #print x data = x.split('/')[-1] if '._' in data: continue data2 = data.split('_') #print data2[0] #print data2[1][:-4] key = data2[0] i = data2[1][:-4] #print chi #print key if key not in chi: continue #print chi #print key In = 1 name = "http://www.columbia.edu/AIDA/DVMM/Entities/FaceDetection/RUN00010/" + str( key) + '/' + str(i) #entityList.append(key) #entityList.append(entity) entity = aifutils.make_entity(g, name, sys) first_cluster = 1 #======================== JPG =============== eid_list = [] if key in OD_result.keys(): for n in range(len(OD_result[key])): #print OD_result[key][n]['label'] if OD_result[key][n]['label'] in person_label: #print OD_result[key][n]['label'] boxA = OD_result[key][n]['bbox'] boxB = (int(bb[x][0]), int(bb[x][1]), int(bb[x][2]), int(bb[x][3])) IOA = bb_intersection_over_union(boxA, boxB) if IOA > 0.9: #left,top,right,bottom = OD_result[key][n]['bbox'] #if int(bb[x][1]) > left and int(bb[x][0]) > top and int(bb[x][3]) < right and int(bb[x][2]) < bottom: eid = "http://www.columbia.edu/AIDA/DVMM/Entities/ObjectDetection/RUN00010/JPG/" + key + "/" + str( n) #print entity_dic2[key] #print n if n in entity_dic2[key]: #if eid in entity_dict.keys(): score = IOA eid_list.append(eid) if first_cluster == 1: first_cluster = 0 clusterName = aifutils.make_cluster_with_prototype(g, \ "http://www.columbia.edu/AIDA/DVMM/Clusters/HumanBody/RUN00010/JPG/"\ +key+"/"+str(n)+"/"+str(person_c_n),entity, sys) aifutils.mark_as_possible_cluster_member(g, \ entity_dict[eid],clusterName, score, sys) if first_cluster == 0: person_c_n += 1 l, t, r, d = bb[x] if (r - l) * (d - t) > 3600: entityList.append(entity) arrayList.append(y[2]) feature = {} feature['columbia_vector_faceID_FaceNet'] = y[2].tolist() json_data = json.dumps(feature) aifutils.mark_private_data(g, entity, json_data, sys) #labelrdf = VIS_ONTOLOGY.term(i_id) #Dscore = value[i][7] #if Dscore>1: Dscore = 1 #type_assertion = aifutils.mark_type(g, "Columbia/DVMM/TypeAssertion/FaceRecognition/RUN00003/"+str(i_id)+"/"+str(i)+"/1", type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/FaceDetection/RUN00010/JPG/"+\ str(key)+'/'+str(i), entity, AIDA_PROGRAM_ONTOLOGY2.term('PER'), sys, Dscore) bb2 = Bounding_Box((bb[x][0], bb[x][1]), (bb[x][2], bb[x][3])) justif = aifutils.mark_image_justification(g, [entity, type_assertion], key, bb2, sys, 1) aifutils.add_source_document_to_justification(g, justif, parent) aifutils.mark_informative_justification(g, entity, justif) chi_set.add(key) parent_set.add(parent) if y[0].replace(' ', '_') not in nameSet or nameDic[y[0].replace( ' ', '_')] not in p1Set or float(y[1]) < 0.04 or y[0].replace( ' ', '_') == 'Ban_Ki-moon': continue else: #nameCount2+=1 person_set.add(y[0]) doc_N.add(parent) img_N.add(key) score = sigmoid(float(y[1]) * 10) NameDetected.add(nameDic[y[0].replace(' ', '_')]) #print y[0] entity_key = nameDic[y[0].replace(' ', '_')] type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/FaceID/"\ +str(index_category[entity_key])+'/'+entity_key, entityDic[entity_key], AIDA_PROGRAM_ONTOLOGY2.term('PER'), sys, 1) justif = aifutils.mark_image_justification(g, [entityDic[nameDic[y[0].replace(' ','_')]], \ type_assertion], key, bb2, sys, score) aifutils.add_source_document_to_justification(g, justif, parent) aifutils.mark_informative_justification( g, entityDic[nameDic[y[0].replace(' ', '_')]], justif) # entity, labelrdf, sys, score) #print str(value[i][2]).replace("L",'') #========================== Video Frame ================ for x, y in result2.items(): #print x data = x.split('/')[-2] if '._' in data: continue data = x.split('/')[-1] data2 = data.split('_') key = x.split('/')[-2] if videoDic[key] not in chi: continue #print "video" In = 1 i = data2[-1][:-4] frame = data[:-len(data2[-1]) - 1] frameNum = frame.split('_')[-1] name = "http://www.columbia.edu/AIDA/DVMM/Entities/FaceDetection/RUN00010/Keyframe/"+\ str(videoDic[key])+'_'+str(frameNum)+'/'+str(i) entity = aifutils.make_entity(g, name, sys) if str(videoDic[key]) + '_' + str(frameNum) in ODF_result.keys(): first_cluster = 1 for n in range( len(ODF_result[str(videoDic[key]) + '_' + str(frameNum)])): if ODF_result[str(videoDic[key]) + '_' + str(frameNum)][n]['label'] in person_label: boxA = ODF_result[str(videoDic[key]) + '_' + str(frameNum)][n]['bbox'] boxB = (int(bb_2[x][0]), int(bb_2[x][1]), int(bb_2[x][2]), int(bb_2[x][3])) IOA = bb_intersection_over_union(boxA, boxB) if IOA > 0.9: eid = "http://www.columbia.edu/AIDA/DVMM/Entities/ObjectDetection/RUN00010/Keyframe/" + str( videoDic[key]) + '_' + str(frameNum) + "/" + str(n) if n in entity_dic2[str(videoDic[key]) + '_' + str(frameNum)]: #if eid in entity_dict.keys(): score = IOA #print x #print entity_dict[eid] #print n if first_cluster == 1: first_cluster = 0 clusterName = aifutils.make_cluster_with_prototype(g, \ "http://www.columbia.edu/AIDA/DVMM/Clusters/HumanBody/RUN00010/Keyframe/"+\ str(videoDic[key])+'_'+str(frameNum)+'/'+str(i)+'/'+\ str(person_c_n),entity, sys) #aifutils.mark_as_possible_cluster_member(g, \ # entity,clusterName, score, sys) aifutils.mark_as_possible_cluster_member(g, \ entity_dict[eid],clusterName, score, sys) if first_cluster == 0: person_c_n += 1 #txn.put("Columbia/DVMM/TypeAssertion/FaceID/RUN00003/"+str(key)+'/'+str(i), value[i][4]); #featureDic[entity] = y[2] featureDic[key] = y[2] #entityList.append(key) l, t, r, d = bb_2[x] if (r - l) * (d - t) > 3600: entityList.append(entity) arrayList.append(y[2]) #if first == 1: # new_array = [y[2]] # first = 0 #else: # new_array = np.concatenate((new_array, [y[2]]), axis=0) feature = {} feature['columbia_vector_faceID_FaceNet'] = y[2].tolist() json_data = json.dumps(feature) aifutils.mark_private_data(g, entity, json_data, sys) #labelrdf = VIS_ONTOLOGY.term(i_id) #Dscore = value[i][7] #if Dscore>1: Dscore = 1 #type_assertion = aifutils.mark_type(g, "Columbia/DVMM/TypeAssertion/FaceRecognition/RUN00003/"+str(i_id)+"/"+str(i)+"/1", type_assertion = aifutils.mark_type(g, "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/FaceDetection/RUN00010/Keyframe/"+\ str(videoDic[key])+'_'+str(frameNum)+'/'+str(i), entity, AIDA_PROGRAM_ONTOLOGY2.term('PER'), sys, Dscore) #str(videoDic[key])+'_'+str(frameNum)+'/'+str(i), entity, AIDA_PROGRAM_ONTOLOGY2.Entity, sys, Dscore) #print bb[x][1] bb2 = Bounding_Box((bb_2[x][0], bb_2[x][1]), (bb_2[x][2], bb_2[x][3])) #aifutils.mark_image_justification(g, [entity, type_assertion], key, bb2, sys, 1) justif = aifutils.mark_keyframe_video_justification(g, [entity, type_assertion], videoDic[key], \ str(videoDic[key])+'_'+str(frameNum), bb2, sys, 1) aifutils.add_source_document_to_justification(g, justif, parent) aifutils.mark_informative_justification(g, entity, justif) chi_set.add(key) parent_set.add(parent) if y[0].replace(' ', '_') not in nameSet or nameDic[y[0].replace( ' ', '_')] not in p1Set or float(y[1]) < 0.04 or y[0].replace( ' ', '_') == 'Ban_Ki-moon': continue else: #nameCount2+=1 person_set.add(y[0]) doc_N.add(parent) img_N.add(key) #if float(y[1])*10>1: # score = 1-random.random()/10 #else: # score = float(y[1])*10 score = sigmoid(float(y[1]) * 10) #place_of_birth_in_louisville_cluster = aifutils.mark_as_possible_cluster_member(g, \ # entity,clusterDic[nameDic[y[0].replace(' ','_')]], score, sys) NameDetected.add(nameDic[y[0].replace(' ', '_')]) entity_key = nameDic[y[0].replace(' ', '_')] type_assertion = aifutils.mark_type(g, \ "http://www.columbia.edu/AIDA/DVMM/TypeAssertion/FaceID/"\ +str(index_category[entity_key])+'/'+entity_key, entityDic[entity_key], AIDA_PROGRAM_ONTOLOGY2.term('PER'), sys, 1) justif = aifutils.mark_keyframe_video_justification(g, [entityDic[nameDic[y[0].replace(' ','_')]], type_assertion], videoDic[key], \ str(videoDic[key])+'_'+str(frameNum), bb2, sys, score) aifutils.add_source_document_to_justification(g, justif, parent) aifutils.mark_informative_justification( g, entityDic[nameDic[y[0].replace(' ', '_')]], justif) #dbscan_run(arrayList,entityList) new_array = np.array(arrayList) if len(arrayList) > 1: # Compute DBSCAN #if __name__ == '__main__': db = DBSCAN(eps=0.55, min_samples=2).fit(new_array) core_samples_mask = np.zeros_like(db.labels_, dtype=bool) core_samples_mask[db.core_sample_indices_] = True labels = db.labels_ #print labels # Number of clusters in labels, ignoring noise if present. n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) #print entityList #print('Estimated number of clusters: %d' % n_clusters_) clusterNameDic = {} firstMem = [0 for i in range(n_clusters_)] firstArray = {} for i in range(len(labels)): if labels[i] == -1: continue #print len(labels) #print len(entityList) #score = 1 if firstMem[labels[i]] == 0: firstMem[labels[i]] = 1 firstArray[labels[i]] = new_array[i] clusterNameDic[labels[i]] = aifutils.make_cluster_with_prototype(g, \ "http://www.columbia.edu/AIDA/DVMM/Clusters/FaceCoreference/RUN00010/"+\ str(labels[i]),entityList[i], sys) #print entityList[a][j] else: dist = np.linalg.norm(firstArray[labels[i]] - new_array[i]) if dist > 1: score = 0.001 else: score = 1 - dist / 2 #score = sigmoid(dist) #print score aifutils.mark_as_possible_cluster_member(g, \ entityList[i],clusterNameDic[labels[i]], score, sys) sys = aifutils.make_system_with_uri( g, "http://www.columbia.edu/AIDA/DVMM/Systems/Face/FaceNet") for key, value in index_category.items(): if key not in NameDetected: continue key = key.replace(' ', '_') #print name2ID[] new_key = key.lower().replace('_', ' ') #print new_key try: #print RPI[parent].keys() #print name2ID[new_key] if name2ID[new_key] in RPI[parent].keys(): print(new_key) #print parent cluster = aifutils.make_cluster_with_prototype(g, \ "http://www.columbia.edu/AIDA/DVMM/Clusters/NamedPersonCoreference/"+\ str(value)+'/'+key,entityDic[key], sys) score = 1 #aifutils.mark_as_possible_cluster_member(g, ,cluster, score, sys) for i in range(len(RPI[parent][name2ID[new_key]])): aifutils.mark_as_possible_cluster_member(g, \ RPI[parent][name2ID[new_key]][i],cluster, score, sys) except KeyError: a = 0 directory = ttl_out + '/' if not os.path.exists(directory): os.makedirs(directory) with open(directory + parent + '.ttl', 'w') as fout: serialization = BytesIO() # need .buffer because serialize will write bytes, not str g.serialize(destination=serialization, format='turtle') fout.write(serialization.getvalue().decode('utf-8'))