def find_propert_time(alist: Alist):
alist_arr = []
results = find_recording(artist=alist.get(tt.SUBJECT),
title=alist.get(tt.OBJECT),
date=None)
# parse date formats and sort in reverse
FORMATS = ['%Y', '%Y-%m-%d']
for r in results:
date = ''
for fmt in FORMATS:
try:
date = datetime.strptime(r['date'], fmt)
r['date'] = date.strftime('%Y')
except:
pass
results_sorted = [k for k in sorted(results, key=lambda x: x['date'])]
for item in results_sorted:
data_alist = alist.copy()
data_alist.set(tt.TIME, item['date'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['musicbrainz']))
alist_arr.append(data_alist)
break # greedy; take only the first answer returned
return alist_arr
def decompose(self, alist: A, G: InferenceGraph):
# check for comparison operations: eq, lt, gt, lte, gte and for multiple variables in operation variable
if alist.get(tt.OP).lower() in ['eq', 'lt', 'gt', 'lte', 'gte'] \
and len(alist.get(tt.OPVAR).split(' ')) > 1:
opvars = alist.get(tt.OPVAR).split(' ')
op_alist = alist.copy()
# higher cost makes this decomposition more expensive
op_alist.cost = alist.cost + 1
op_alist.branch_type = br.OR
op_alist.parent_decomposition = 'comparison'
op_alist.node_type = nt.HNODE
# op_alist.state = states.EXPLORED
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
for p in opvars:
pval = alist.get(p)
child = Alist()
child.set(tt.OP, "value")
child.set(tt.OPVAR, p)
child.set(p, pval)
child.cost = op_alist.cost + 1
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
# op_alist.link_child(child)
G.link(op_alist, child, op_alist.parent_decomposition)
else:
return None
return op_alist
def part_of_relation_subject(alist: Alist):
results = []
for r in find_relation_subject(alist.get(tt.OBJECT), "location"):
factAlist = alist.copy()
factAlist.data_sources.add('wikidata')
factAlist.set(tt.SUBJECT, r)
results.append(factAlist)
return results
def part_of_relation_object(alist: Alist):
results = []
for r in _part_of_relation_object(alist.get(tt.SUBJECT), "location"):
fact_alist = alist.copy()
fact_alist.data_sources = list(
set(fact_alist.data_sources + ['wikidata']))
fact_alist.set(tt.OBJECT, r)
results.append(fact_alist)
return results
def part_of_geopolitical_subject(alist: Alist):
results = []
geopolitical_type = alist.get(tt.PROPERTY).split(':')
for r in find_geopolitical_subelements(alist.get(tt.OBJECT),
geopolitical_type[-1]):
fact_alist = alist.copy()
fact_alist.data_sources = list(
set(fact_alist.data_sources + ['wikidata']))
fact_alist.set(tt.SUBJECT, r)
results.append(fact_alist)
return results
def find_property_subject(alist: Alist):
alist_arr = []
results = find_recording(artist=None,
title=alist.get(tt.OBJECT),
date=alist.get(tt.TIME))
for item in results:
data_alist = alist.copy()
data_alist.set(tt.SUBJECT, item['artist'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['musicbrainz']))
alist_arr.append(data_alist)
return alist_arr
def decompose(self, alist: A, G: InferenceGraph):
# check if subject is empty or is a variable
if not alist.get(tt.SUBJECT) \
or alist.get(tt.SUBJECT).startswith(vx.PROJECTION) \
or alist.get(tt.SUBJECT).startswith(vx.AUXILLIARY):
return None
# get the sub locations of the subject
# TODO: perform geospatial decomp on OBJECT attribute
sub_items = sparqlEndpoint.find_sub_location(
alist.get(tt.SUBJECT).strip())
if not sub_items:
return None
alist.data_sources.add('geonames')
op_alist = alist.copy()
op_alist.set(tt.OP, 'sum')
# higher cost makes this decomposition more expensive
op_alist.cost = alist.cost + 4
op_alist.branch_type = br.AND
op_alist.parent_decomposition = 'geospatial'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
for s in sub_items:
child = alist.copy()
child.set(tt.SUBJECT, s)
child.set(tt.OP, 'value')
child.cost = op_alist.cost + 1
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
# op_alist.link_child(child)
G.link(op_alist, child, op_alist.parent_decomposition)
return op_alist
def find_property_values(self, alist: Alist, search_element: str):
if search_element == tt.OBJECT:
subject = alist.instantiation_value(tt.SUBJECT)
nodes = self._get_nodes(subject)
results = []
for node in nodes:
try:
data_alist = alist.copy()
data_alist.set(tt.OBJECT, node[alist.get(tt.PROPERTY)])
data_alist.data_sources = list(
set(data_alist.data_sources + [self.name]))
results.append(data_alist)
except:
pass
return results
def find_property_subject(alist: Alist):
entity_id = find_entity(alist.instantiation_value(tt.OBJECT),
alist.get(tt.PROPERTY))
if not entity_id:
return []
# compose wikidata query
query = ""
if alist.get(tt.TIME):
query = """
SELECT DISTINCT ?sLabel (YEAR(?date) as ?year) WHERE{{
?s wdt:{property_id} wd:{entity_id}.
OPTIONAL {{wd:{entity_id} pq:P585 ?date .}}
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en".}} }
}
""".format(entity_id=entity_id,
property_id=alist.get(tt.PROPERTY))
else:
query = """
SELECT DISTINCT ?s ?sLabel WHERE {{
OPTIONAL {{ ?s wdt:{property_id} wd:{entity_id} . }}
OPTIONAL {{ wd:{entity_id} wdt:{property_id} ?s . }} # hack to find inverse triple
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en".}}
}}
""".format(entity_id=entity_id,
property_id=alist.get(tt.PROPERTY))
params = {'format': 'json', 'query': query}
response = requests.get(url='https://query.wikidata.org/sparql',
params=params)
alist_arr = []
try:
data = response.json()
for d in data['results']['bindings']:
data_alist = alist.copy()
data_alist.set(tt.SUBJECT, d['sLabel']['value'])
if 'year' in d:
data_alist.set(tt.TIME, d['year']['value'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['wikidata']))
alist_arr.append(data_alist)
except Exception as e:
print("wikidata query response error: " + str(e))
return alist_arr
def find_property_object(alist: Alist):
results = []
subj_instantiation = alist.instantiation_value(tt.SUBJECT)
if isinstance(subj_instantiation, str):
country_id = getCountryPropertyDb(subj_instantiation.replace("_", " "),
"id")
else:
return results
if not country_id:
return results
try:
params = {
'date': str(alist.get(tt.TIME)).replace(".0", ""),
'format': 'json',
'per_page': 1000
}
response = requests.get(
url=
f'http://api.worldbank.org/v2/countries/{country_id}/indicators/{alist.get(tt.PROPERTY)}',
params=params)
try:
data = response.json()
if len(data) > 1 and data[1]:
for d in data[1]:
if d['value']:
data_alist = alist.copy()
data_alist.set(tt.OBJECT, d['value'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['worldbank']))
results.append(data_alist)
except Exception as ex:
print("worldbank query response error: " + str(ex))
except Exception as ex:
print("worldbank query error: " + str(ex))
return results
def decompose(self, alist: A, G: InferenceGraph):
current_year = datetime.datetime.now().year
branch_factor = config.config["temporal_branching_factor"]
parent_year = None
if alist.get(tt.TIME).startswith(vx.NESTING) or \
not alist.get(tt.TIME):
return None
else:
parent_year = datetime.datetime.strptime(alist.get(tt.TIME), '%Y')
count = 0
op_alist = alist.copy()
op = "regress"
context = op_alist.get(tt.CONTEXT)
if context:
if context[0]:
if ctx.accuracy in context[0] and context[0][ctx.accuracy] == 'high':
op = 'gpregress'
if branch_factor <= 10:
# increase number of data points for regression
branch_factor = 20
# if context[1] and ctx.datetime in context[1]:
# # use the ctx.datetime as current year if specified in context
# current_year = datetime.datetime.strptime(context[1][ctx.datetime], '%Y-%m-%d %H:%M:%S').year
# flush context: needed to clear any query time context value
# whose corresponding alist attribute (t) has been modified
frank.context.flush(op_alist, [tt.TIME])
op_alist.set(tt.OP, op)
op_alist.cost = alist.cost + 2.0
op_alist.branch_type = br.AND
op_alist.state = states.EXPLORED
op_alist.parent_decomposition = 'temporal'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
if (current_year - parent_year.year) > branch_factor/2:
for i in range(1, math.ceil(branch_factor/2)):
child_a = alist.copy()
child_a.set(tt.TIME, str(parent_year.year + i))
child_a.set(tt.OP, "value")
child_a.cost = op_alist.cost + 1
child_a.node_type = nt.ZNODE
child_a.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
frank.context.flush(child_a, [tt.TIME])
# op_alist.link_child(child_a)
G.link(op_alist, child_a, 'value')
child_b = alist.copy()
child_b.set(tt.TIME, str(parent_year.year - i))
child_b.set(tt.OP, "value")
child_b.cost = op_alist.cost + 1
child_b.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
frank.context.flush(child_b, [tt.TIME])
child_b.node_type = nt.ZNODE
# op_alist.link_child(child_b)
G.link(op_alist, child_b, 'value')
count = count + 2
elif parent_year.year >= current_year:
for i in range(1, math.ceil(branch_factor)):
child_a = alist.copy()
child_a.set(tt.TIME, str(current_year - i))
child_a.set(tt.OP, "value")
child_a.cost = op_alist.cost + 1
child_a.node_type = nt.ZNODE
child_a.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
frank.context.flush(child_a, [tt.TIME])
# op_alist.link_child(child_a)
G.link(op_alist, child_a, 'value')
count = count + 1
for i in range(1, (branch_factor - count)):
child_a = alist.copy()
child_a.set(tt.TIME, str(parent_year.year - (count + i)))
child_a.set(tt.OP, "value")
child_a.cost = op_alist.cost + 1
child_a.node_type = nt.ZNODE
child_a.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
frank.context.flush(child_a, [tt.TIME])
# op_alist.link_child(child_a)
G.link(op_alist, child_a, 'value')
return op_alist
def decompose(self, alist: A, G: InferenceGraph):
nest_vars = alist.uninstantiated_nesting_variables()
for nest_attr, v in nest_vars.items():
if NormalizeFn.FILTER in v:
op_alist = alist.copy()
op_alist.set(tt.OPVAR, nest_attr)
op_alist.set(tt.OP, 'comp')
del op_alist.attributes[nest_attr]
op_alist.cost = alist.cost + 1
op_alist.branch_type = br.AND
op_alist.state = states.EXPLORED
op_alist.parent_decomposition = 'normalize'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
# check for filters that heuristics apply to
# e.g type==country and location==Europs
filter_patterns = {}
geo_class = ''
for x in v[NormalizeFn.FILTER]:
prop = str(x['p'])
obj = str(x['o'])
if prop == 'type' and (obj == 'country'
or obj == 'continent'):
filter_patterns['geopolitical'] = obj
elif prop == 'location':
filter_patterns['location'] = obj
if {'geopolitical', 'location'} <= set(filter_patterns):
# use heuristics to create a single alist containing the
# conjunction to find the X located in Y
child = A(**{})
child.set(tt.OP, 'values')
child.set(tt.OPVAR, nest_attr)
child.set(tt.SUBJECT, nest_attr)
child.set(
tt.PROPERTY,
'__geopolitical:' + filter_patterns['geopolitical'])
child.set(tt.OBJECT, filter_patterns['location'])
child.cost = op_alist.cost + 1
child.state = states.UNEXPLORED
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
child = frank.context.inject_query_context(child)
G.link(op_alist, child, op_alist.parent_decomposition)
return op_alist
else:
for x in v[NormalizeFn.FILTER]:
child = A(**{})
child.set(tt.OP, 'values')
child.set(tt.OPVAR, nest_attr)
child.set(tt.SUBJECT, nest_attr)
for attr, attrval in x.items():
child.set(attr, attrval)
child.cost = op_alist.cost + 1
child.state = states.UNEXPLORED
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
child = frank.context.inject_query_context(child)
G.link(op_alist, child, op_alist.parent_decomposition)
return op_alist
elif NormalizeFn.IN in v:
op_alist = alist.copy()
op_alist.set(tt.OPVAR, nest_attr)
op_alist.set(tt.OP, 'comp')
del op_alist.attributes[nest_attr]
op_alist.cost = alist.cost + 1
op_alist.state = states.EXPLORED
op_alist.parent_decomposition = 'normalize'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
listed_items = []
if isinstance(v[NormalizeFn.IN], list):
for x in v[NormalizeFn.IN]:
listed_items.append(str(x))
elif isinstance(v[NormalizeFn.IN], str):
for x in str(v[NormalizeFn.IN]).split(';'):
listed_items.append(str(x).strip())
for x in listed_items:
child = A(**{})
child.set(tt.OP, 'value')
if nest_attr[0] in [
vx.AUXILLIARY, vx.PROJECTION, vx.NESTING
]:
child.set(tt.OPVAR, nest_attr)
child.set(nest_attr, x)
else:
new_var = vx.PROJECTION + '_x' + \
str(len(op_alist.attributes))
child.set(tt.OPVAR, new_var)
child.set(nest_attr, new_var)
child.set(new_var, x)
child.state = states.UNEXPLORED
child.node_type = nt.ZNODE
child.cost = op_alist.cost + 1
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
child = frank.context.inject_query_context(child)
G.link(op_alist, child, op_alist.parent_decomposition)
return op_alist
elif NormalizeFn.IS in v:
op_alist = alist.copy()
op_alist.set(tt.OPVAR, nest_attr)
op_alist.set(tt.OP, 'comp')
del op_alist.attributes[nest_attr]
op_alist.cost = alist.cost + 1
op_alist.state = states.EXPLORED
op_alist.parent_decomposition = 'normalize'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
child = A(**{})
child.set(tt.OP, 'value')
new_var = vx.PROJECTION + '_x' + str(len(op_alist.attributes))
child.set(tt.OPVAR, new_var)
child.set(new_var, v[NormalizeFn.IS])
child.state = states.REDUCIBLE
child.cost = op_alist.cost + 1
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
child = frank.context.inject_query_context(child)
G.link(op_alist, child, op_alist.parent_decomposition)
if v[NormalizeFn.IS].startswith(
(vx.AUXILLIARY, vx.NESTING, vx.PROJECTION)) == False:
# this is an instantiation, so a pseudo leaf node should be created
leaf = A(**{})
leaf.set(tt.OP, 'value')
new_var = vx.PROJECTION + '_x' + \
str(len(op_alist.attributes))
leaf.set(tt.OPVAR, new_var)
leaf.set(new_var, v[NormalizeFn.IS])
leaf.state = states.REDUCIBLE
leaf.cost = op_alist.cost + 1
leaf.node_type = nt.ZNODE
leaf.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
leaf = frank.context.inject_query_context(leaf)
G.link(child, leaf, op_alist.parent_decomposition)
return op_alist
elif tt.OP in v:
op_alist = alist.copy()
op_alist.set(tt.OPVAR, nest_attr)
op_alist.set(tt.OP, 'comp')
# del op_alist.attributes[nest_attr]
op_alist.set(nest_attr, '')
op_alist.cost = alist.cost + 1
op_alist.parent_decomposition = 'normalize'
op_alist.node_type = nt.HNODE
# alist.link_child(op_alist)
G.link(alist, op_alist, op_alist.parent_decomposition)
var_ctr = 200
child = A(**{})
for ak, av in v.items():
if isinstance(av, str):
child.set(ak, av.strip())
elif ak == tt.CONTEXT:
child.set(ak, av)
else:
new_var = vx.NESTING + str(var_ctr)
child.set(ak, new_var)
child.set(new_var, av)
var_ctr = var_ctr + 1
child.cost = op_alist.cost + 1
child.node_type = nt.ZNODE
child.set(tt.CONTEXT, op_alist.get(tt.CONTEXT))
child = frank.context.inject_query_context(child)
G.link(op_alist, child, op_alist.parent_decomposition)
return op_alist
return None
def find_property_object(alist: Alist):
entity_id = None
wikidata_base_uri = 'http://www.wikidata.org/entity/'
if 'http://www.wikidata.org/entity/' in alist.instantiation_value(
tt.SUBJECT):
entity_id = alist.instantiation_value(
tt.SUBJECT)[len(wikidata_base_uri):]
else:
entity_id = find_entity(alist.instantiation_value(tt.SUBJECT),
alist.get(tt.PROPERTY))
if not entity_id:
return []
# compose wikidata query
query = """
SELECT DISTINCT ?oLabel (YEAR(?date) as ?year) WHERE {{
wd:{entity_id} p:{property_id} ?ob .
?ob ps:{property_id} ?o .
OPTIONAL {{ ?ob pq:P585 ?date . }}
OPTIONAL {{ ?ob pq:P580 ?date . }}
OPTIONAL {{ FILTER (YEAR(?date) = {time}) . }}
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }} }}
ORDER By DESC(?year)
""".format(entity_id=entity_id,
property_id=alist.get(tt.PROPERTY),
time=alist.get(tt.TIME).replace(".0", "")
if alist.get(tt.TIME) else '0')
params = {'format': 'json', 'query': query}
response = requests.get(url='https://query.wikidata.org/sparql',
params=params)
alist_arr = []
try:
data = response.json()
ctx = {}
if data['results']['bindings']:
ctx = alist.get(tt.CONTEXT)
ctx = {**ctx[0], **ctx[1], **ctx[2]} if ctx else {}
result_with_year = False
for d in data['results']['bindings']:
if 'year' in d:
result_with_year = True
break
for d in data['results']['bindings']:
# if alist has explicit time and no context,
# or has explicit time not from context
# then result must include time
if (alist.get(tt.TIME) and tt.TIME not in ctx):
if ('year' in d) and (d['year']['value'] == alist.get(
tt.TIME)):
data_alist = alist.copy()
data_alist.set(tt.OBJECT, d['oLabel']['value'])
data_alist.set(tt.TIME, d['year']['value'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['wikidata']))
alist_arr.append(data_alist)
# else if time is injected from context
# then only append results that have no time only if the dataset is empty..
# wikidata returns bindings with the time attribute first so this works
elif alist.get(tt.TIME) and tt.TIME in ctx:
current_year = str(datetime.now().year)
if (('year' in d) and (d['year']['value'] == alist.get(tt.TIME))) or \
((((('year' in d) and (d['year']['value'] != alist.get(tt.TIME))) and len(alist_arr) == 0) or
(('year' not in d) and len(alist_arr) == 0)) and
(
(alist.get(tt.TIME) == current_year and (not frank.util.utils.is_numeric(d['oLabel']['value']))) or
(alist.get(tt.TIME) ==
ctx[tt.TIME] and result_with_year == False)
)):
# last condition: append this value only if time (i.e the context time) is the current year and the data value is not numeric.
data_alist = alist.copy()
data_alist.set(tt.OBJECT, d['oLabel']['value'])
# if 'year' in d: # use time in dataset optionally
# data_alist.set(tt.TIME, d['year']['value'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['wikidata']))
alist_arr.append(data_alist)
else:
data_alist = alist.copy()
data_alist.set(tt.OBJECT, d['oLabel']['value'])
# if 'year' in d: # use time in dataset optionally
# data_alist.set(tt.TIME, d['year']['value'])
data_alist.data_sources = list(
set(data_alist.data_sources + ['wikidata']))
alist_arr.append(data_alist)
except Exception as ex:
print("wikidata query response error: " + str(ex))
return alist_arr
def search_kb(self, alist: Alist):
""" Search knowledge bases to instantiate variables in alist.
Args
----
alist: Alist
Return
------
Returns `True` if variable instantiation is successful from a KB search.
"""
self.last_heartbeat = time.time()
prop_refs = []
found_facts = []
# cannot search if alist has uninstantiated nested variables
if alist.uninstantiated_nesting_variables():
return found_facts
self.write_trace(
f"{pcol.MAGENTA}search {alist.id}{pcol.RESET} {alist}{pcol.RESETALL}"
)
if alist.state == states.EXPLORED:
new_alist = alist.copy()
new_alist.state = states.EXPLORED
new_alist.set(tt.OPVAR, alist.get(tt.OPVAR))
return True
prop_string = alist.get(tt.PROPERTY)
sources = {
'wikidata': {
'fn': wikidata,
'trust': 'low'
},
'worldbank': {
'fn': worldbank,
'trust': 'high'
},
'musicbrainz': {
'fn': musicbrainz,
'trust': 'high'
}
}
# ,
# 'gregbrimblecom!': {'fn': jsonld.JSONLD.from_url('gregbrimblecom!', 'https://gregbrimble.com'), 'trust': 'high'},
# 'mozilla': {'fn': jsonld.JSONLD.from_url('mozilla', 'https://www.mozilla.org/en-GB/'), 'trust': 'high'}
# }
context = alist.get(tt.CONTEXT)
context_store = {}
context_store = {
**context[0],
**context[1],
**context[2]
} if context else {}
for source_name, source in sources.items():
# check context for trust
if ctx.trust in context_store:
if context_store[
ctx.trust] == 'high' and source['trust'] != 'high':
continue
# for source_name, source in {'worldbank':worldbank}.items():
search_alist = alist.copy()
# inject context into IR
search_alist = frank.context.inject_retrieval_context(
search_alist, source_name)
# if the property_refs does not contain an entry for the property in this alist
# search KB for a ref for the property
prop_sources = []
if prop_string in self.property_refs:
prop_sources = [x[1] for x in self.property_refs[prop_string]]
if (prop_string not in self.property_refs and not prop_string.startswith('__')) \
or (prop_string in self.property_refs and source_name not in prop_sources):
props = source['fn'].search_properties(prop_string)
if len(props) > 0:
maxScore = 0
for p in props:
if p[2] >= maxScore:
prop_refs.append((p, source_name))
self.reverse_property_refs[p[0]] = prop_string
maxScore = p[2]
else:
break
self.property_refs[prop_string] = prop_refs
search_attr = tt.SUBJECT
uninstantiated_variables = search_alist.uninstantiated_attributes()
if tt.SUBJECT in uninstantiated_variables:
search_attr = tt.SUBJECT
elif tt.OBJECT in uninstantiated_variables:
search_attr = tt.OBJECT
elif tt.TIME in uninstantiated_variables:
search_attr = tt.TIME
cache_found_flag = False
if config.config['use_cache']:
searchable_attr = list(
filter(lambda x: x != search_attr,
[tt.SUBJECT, tt.PROPERTY, tt.OBJECT, tt.TIME]))
# search with original property name
(cache_found_flag, results) = (False, [])
# (cache_found_flag, results) = frank.cache.neo4j.search_cache(alist_to_instantiate=search_alist,
# attribute_to_instantiate=search_attr,
# search_attributes=searchable_attr)
if cache_found_flag == True:
found_facts.append(results[0])
# search with source-specific property IDs
for (propid, _source_name) in self.property_refs[prop_string]:
self.last_heartbeat = time.time()
search_alist.set(tt.PROPERTY, propid[0])
(cache_found_flag, results) = (False, [])
# = frank.cache.neo4j.search_cache(alist_to_instantiate=search_alist,
# attribute_to_instantiate=search_attr,
# search_attributes=searchable_attr)
if cache_found_flag == True:
found_facts.append(results[0])
self.write_trace(
f'{pcol.MAGENTA}found: cache{pcol.RESETALL}')
# if not found_facts:
# self.write_trace('found:>>> cache')
if not cache_found_flag and prop_string in self.property_refs:
# search for data for each property reference source
for propid_label, _source_name in self.property_refs[
prop_string]:
self.last_heartbeat = time.time()
try:
if _source_name == source_name:
search_alist.set(tt.PROPERTY, propid_label[0])
found_facts.extend(
source['fn'].find_property_values(
search_alist, search_attr))
# TODO: handle location search in less adhoc manner
if alist.get(tt.PROPERTY).lower() == "location":
if search_attr == tt.SUBJECT:
found_facts.extend(
wikidata.part_of_relation_subject(
search_alist))
elif search_attr == tt.OBJECT:
found_facts.extend(
wikidata.part_of_relation_object(
search_alist))
break
except Exception as ex:
self.write_trace(
f"{pcol.RED}Search Error{pcol.RESETALL}",
processLog.LogLevel.ERROR)
print(str(ex))
if not found_facts and alist.get(
tt.PROPERTY).startswith('__geopolitical:'):
if search_attr == tt.SUBJECT:
found_facts.extend(
wikidata.part_of_geopolitical_subject(search_alist))
# TODO: save facts found to cache if caching is enabled
# if foundFacts and config.config['use_cache']:
# for ff in foundFacts:
# cache().save(ff, ff.dataSources[0])
if found_facts:
self.last_heartbeat = time.time()
all_numeric = True
non_numeric_data_items = []
numeric_data_items = []
for ff in found_facts:
self.last_heartbeat = time.time()
if utils.is_numeric(ff.get(search_attr)):
numeric_data_items.append(
utils.get_number(ff.get(search_attr), 0.0))
else:
all_numeric = False
non_numeric_data_items.append(ff.get(search_attr))
ff.set(tt.OPVAR, alist.get(tt.OPVAR))
ff.set(ff.get(tt.OPVAR), ff.get(search_attr))
sourceCov = sourcePrior().get_prior(
source=list(ff.data_sources)[0]).cov
ff.set(tt.COV, sourceCov)
ff.state = states.REDUCIBLE
ff.set(tt.EXPLAIN, '')
ff.node_type = nt.FACT
if ff.get(tt.PROPERTY) in self.reverse_property_refs:
ff.set(tt.PROPERTY,
self.reverse_property_refs[ff.get(tt.PROPERTY)])
alist.parent_decomposition = "Lookup"
self.G.add_alist(alist)
self.G.link(alist, ff, alist.parent_decomposition)
# fact is considered reduced
self.write_trace(
f' {pcol.MAGENTA}found:{pcol.RESET} {str(ff)}{pcol.RESETALL}'
)
return len(found_facts) > 0
