def test_search(self):
r0 = freebase.search("Kurt V")
self.assertEqual(self.is_kurt_there(r0), True)
r1 = freebase.search("Kurt V", type=["/location/citytown"])
self.assertEqual(self.is_kurt_there(r1), False)
r2 = freebase.search("Kurt V", type=["/location/citytown", "/music/artist"])
self.assertEqual(self.is_kurt_there(r2), False)
self.assertNotEqual(len(r0), len(r1))
self.assertNotEqual(len(r0), len(r2))
self.assertNotEqual(len(r1), len(r2))
def test_search(self):
r0 = freebase.search("Kurt V")
self.assertEqual(self.is_kurt_there(r0), True)
r1 = freebase.search("Kurt V", type=["/location/citytown"])
self.assertEqual(self.is_kurt_there(r1), False)
r2 = freebase.search("Kurt V", type=["/location/citytown", "/music/artist"])
self.assertEqual(self.is_kurt_there(r2), False)
self.assertNotEqual(len(r0), len(r1))
self.assertNotEqual(len(r0), len(r2))
self.assertNotEqual(len(r1), len(r2))
def GetType(self, keyword):
if not keyword:
return [[],[]]
#search for the id
data = freebase.search(keyword, limit=1)
if not data:
return [[],[]]
item_id = [d['id'] for d in data if 'id' in d][0]
query = {"id":item_id, "type":[{}]}
response = freebase.mqlread(query)
types = response['type']
names = []
base_names = []
for k in types:
names.append(k['name'])
type_x = ''.join(k['id']).split('/');
if len(type_x)>1:
base_names.append(type_x[1])
names_set = set(names)
base_names_set = set(base_names)
names = []
base_names = []
for y in names_set:
names.append(y)
for y in base_names_set:
base_names.append(y)
all_list = []
all_list.append(names)
all_list.append(base_names)
return all_list
def GetType(self, keyword):
if not keyword:
return [[], []]
#search for the id
data = freebase.search(keyword, limit=1)
if not data:
return [[], []]
item_id = [d['id'] for d in data if 'id' in d][0]
query = {"id": item_id, "type": [{}]}
response = freebase.mqlread(query)
types = response['type']
names = []
base_names = []
for k in types:
names.append(k['name'])
type_x = ''.join(k['id']).split('/')
if len(type_x) > 1:
base_names.append(type_x[1])
names_set = set(names)
base_names_set = set(base_names)
names = []
base_names = []
for y in names_set:
names.append(y)
for y in base_names_set:
base_names.append(y)
all_list = []
all_list.append(names)
all_list.append(base_names)
return all_list
def search(q, place=None, limit=5):
# TODO: 1 hour cache
if FAKE_MODE:
fb_json = FAKE_FREEBASE_SEARCH
else:
fb_json = freebase.search(q,
type = '/biology/organism_classification',
limit = limit,
mql_output = json.dumps([{
'guid': None,
'name': None,
'/biology/organism_classification/scientific_name': None,
'/common/topic/alias': [],
}])
)
# Add an 'id' field for each result based on the 'guid' field
fb_json = [
dict(item.items() + [
('id', '/guid/%s' % item['guid'].replace('#', ''))
])
for item in fb_json
]
results = _annotate_database_objects(
map(_tidy_up_keys, fb_json), place = place
)
# Custom ordering - things spotted here (or anywhere) take precedence
results.sort(key = lambda o: (
o.get('local_sightings', 0),
o.get('total_sightings', 0),
o.get('relevance:score', 0.0),
), reverse=True)
return results
def freebase_limited_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(type=[
'/base/ontologies/ontology_class', '/freebase/equivalent_topic',
'/base/tagit/concept', '/base/ontologies/ontology_instance',
'/common/topic', '/freebase/equivalent_topic'
],
limit=3,
query=' '.join(tokens))
def get_id(self, search_string, strict_match=False, **args):
if search_string == "" or search_string is None:
return None
relevance = freebase.search(search_string, type=args.get(type))
if len(relevance) == 0:
return None
if strict_match and len(relevance) != 1:
return None
return relevance[0]["id"]
def get_types(self):
'''docs'''
if not self.keyword:
return None
# search for the id
data = freebase.search(self.keyword, limit=1)
if data:
item_id = [d['id'] for d in data if 'id' in d][0]
query = {"id": item_id, "type": [{}], 'key': [], 'name': None}
response = freebase.mqlread(query)
return self.entity_builder(response)
else:
return None
def get_types(self):
'''docs'''
if not self.keyword:
return None
# search for the id
data = freebase.search(self.keyword, limit=1)
if data:
item_id = [d['id'] for d in data if 'id' in d][0]
query = {"id":item_id, "type":[{}], 'key':[], 'name':None}
response = freebase.mqlread(query)
return self.entity_builder(response)
else:
return None
def freebase_limited_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(type=['/base/ontologies/ontology_class', '/freebase/equivalent_topic', '/base/tagit/concept', '/base/ontologies/ontology_instance', '/common/topic', '/freebase/equivalent_topic'], limit=3, query=' '.join(tokens))
def freebase_free_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(query=' '.join(tokens), limit=5)
def run(self):
data = freebase.search(query=self.keyword, limit=3)
if data:
# several ids are necessary as the result of fact that the key search cannot find results
self.result = [d["id"] for d in data if "id" in d]
def run(self):
data = freebase.search(query=self.keyword, limit=3)
if data:
# several ids are necessary as the result of fact that the key search cannot find results
self.result = [d['id'] for d in data if 'id' in d]
def resolve(self, item, test=False):
"""Resolves an entity result of OpenCalais (or other) analysis to a
Freebase-defined entity"""
if not test:
print 'Emily resolution writing to database is not implemented!'
return
cur = self.conn.cursor()
entity = item.data
entity_type = item.type
print 'Resolving', entity
# Check if entity has known type.
# This is done for disambiguation purposes, as it constrains Freebase's search.
resolved_type = self._resolveEntityType(entity_type)
if resolved_type is not None:
print '\tType resolved to', resolved_type
# Send freebase search with resolved type
results = freebase.search(query=entity, type=resolved_type)
# Dictionary keyed by entity, value indicates the number of times this
# entity appears in the database.
counts = {}
# Consider each Freebase entity that was returned in this search for
# disambiguation or inclusion in a semantic group.
for result in results:
eval = self._evaluateEntity(result)
if eval is None:
continue
name, count = eval
if name not in counts:
counts[name] = 0
counts[name] += count
if len(counts) < 1:
print '\tNot enough information to disambiguate against the articles db'
else:
print counts
# Find key with the maximum value
disambig = max(counts.iteritems(), key=operator.itemgetter(1))[0]
print '\tCHOSE:', disambig
# TODO
# We can also improve this method by using existing semantic sources
# about the civil war to ensure accurate disambiguation
# eg. dbpedia - take everything in the 'Civil War' project and
# disambiguate against it.
if test:
print 'Not writing to db'
return
# TODO
# Replace relevant database entries with a single new disambiguated
# entity.
# for entity e in counts
# convert e to disambiguated name where it appears in entity table
# and
# update count column in entity table
for entity in counts:
query = 'UPDATE calais_items WHERE text=? SET text=?'
params = (entity, name)
def freebase_limited_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(type=['/freebase/type_profile', '/type/type'], limit=10, query=' '.join(tokens))
def freebase_free_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(query=' '.join(tokens), limit=7)
def freebase_limited_search(keyword):
tokens = re.findall('[A-Z][^A-Z]*', keyword)
return freebase.search(type=['/freebase/type_profile', '/type/type'],
limit=10,
query=' '.join(tokens))