def test_filter_iterator_interrupt():
expression = "?p = <http://schema.org/eligibleRegion>"
iterator, card = hdtDoc.search_triples(triple['subject'], triple['predicate'], triple['object'])
scan = ProjectionIterator(ScanIterator(iterator, triple, card))
iterator = FilterIterator(scan, expression)
(results, saved, done) = engine.execute(iterator, 10e-7)
assert len(results) <= 4
for b in results:
assert b['?p'] == 'http://schema.org/eligibleRegion'
assert b['?o'] in [
'http://db.uwaterloo.ca/~galuc/wsdbm/Country0',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country1',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country4',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country9'
]
tmp = len(results)
reloaded = load(saved.SerializeToString(), DummyDataset(hdtDoc, 'watdiv100'))
(results, saved, done) = engine.execute(reloaded, 10e7)
assert len(results) + tmp == 4
for b in results:
assert b['?p'] == 'http://schema.org/eligibleRegion'
assert b['?o'] in [
'http://db.uwaterloo.ca/~galuc/wsdbm/Country0',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country1',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country4',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country9'
]
assert done
def test_operation_filter_iterator():
expression = "10 = 5 * 2"
iterator, card = hdtDoc.search_triples(triple['subject'], triple['predicate'], triple['object'])
scan = ProjectionIterator(ScanIterator(iterator, triple, card))
iterator = FilterIterator(scan, expression)
(results, saved, done) = engine.execute(iterator, math.inf)
assert len(results) == 9
def test_function_filter_iterator():
expression = '?p = <http://purl.org/goodrelations/price> && isLiteral(?o) && !isNumeric(?o)'
iterator, card = hdtDoc.search_triples(triple['subject'], triple['predicate'], triple['object'])
scan = ProjectionIterator(ScanIterator(iterator, triple, card))
iterator = FilterIterator(scan, expression)
(results, saved, done) = engine.execute(iterator, math.inf)
assert len(results) == 1
def build_query_plan(query, db_connector, saved_plan=None, projection=None):
cardinalities = []
if saved_plan is not None:
return load(saved_plan, db_connector), []
# optional = query['optional'] if 'optional' in query and len(query['optional']) > 0 else None
root = None
if query['type'] == 'union':
root, cardinalities = build_union_plan(query['union'], db_connector,
projection)
elif query['type'] == 'bgp':
root, cardinalities = build_join_plan(query['bgp'],
db_connector,
projection=projection)
else:
raise Exception('Unkown query type found during query optimization')
# apply filter clause(s)
if 'filters' in query and len(query['filters']) > 0:
# reduce all filters in a conjunctive expression
expression = reduce(lambda x, y: "({}) && ({})".format(x, y),
query['filters'])
root = FilterIterator(root, expression)
return root, cardinalities
def test_and_or_filter_iterator():
expression = "?p = <http://schema.org/eligibleRegion> && (?o = <http://db.uwaterloo.ca/~galuc/wsdbm/Country0> || ?o = <http://db.uwaterloo.ca/~galuc/wsdbm/Country9>)"
iterator, card = hdtDoc.search_triples(triple['subject'], triple['predicate'], triple['object'])
scan = ProjectionIterator(ScanIterator(iterator, triple, card))
iterator = FilterIterator(scan, expression)
(results, saved, done) = engine.execute(iterator, math.inf)
assert len(results) == 2
for b in results:
assert b['?p'] == 'http://schema.org/eligibleRegion'
assert b['?o'] in [
'http://db.uwaterloo.ca/~galuc/wsdbm/Country0',
'http://db.uwaterloo.ca/~galuc/wsdbm/Country9'
]
def parse_query_node(node, dataset, current_graphs, server_url, cardinalities):
"""
Recursively parse node in the query logical plan to build a preemptable physical query execution plan.
Args:
* node - Node of the logical plan to parse (in rdflib format)
* dataset - RDF dataset used to execute the query
* current_graphs - List of IRI of the current RDF graph queried
* server_url - URL of the SaGe server
* cardinalities - Map<triple,integer> used to track triple patterns cardinalities
"""
if node.name == 'SelectQuery':
# in case of a FROM clause, set the new default graphs used
graphs = current_graphs
if node.datasetClause is not None:
graphs = [format_graph_uri(format_term(graph_iri.default), server_url) for graph_iri in node.datasetClause]
return parse_query_node(node.p, dataset, graphs, server_url, cardinalities)
elif node.name == 'Project':
query_vars = list(map(lambda t: '?' + str(t), node._vars))
child = parse_query_node(node.p, dataset, current_graphs, server_url, cardinalities)
return ProjectionIterator(child, query_vars)
elif node.name == 'BGP':
# bgp_vars = node._vars
triples = list(localize_triple(node.triples, current_graphs))
iterator, query_vars, c = build_left_plan(triples, dataset, current_graphs)
# track cardinalities of every triple pattern
cardinalities += c
return iterator
elif node.name == 'Union':
left = parse_query_node(node.p1, dataset, current_graphs, server_url, cardinalities)
right = parse_query_node(node.p2, dataset, current_graphs, server_url, cardinalities)
return BagUnionIterator(left, right)
elif node.name == 'Filter':
expression = parse_filter_expr(node.expr)
iterator = parse_query_node(node.p, dataset, current_graphs, server_url, cardinalities)
return FilterIterator(iterator, expression)
elif node.name == 'Join':
# only allow for joining BGPs from different GRAPH clauses
triples = fetch_graph_triples(node.p1, current_graphs, server_url) + fetch_graph_triples(node.p2, current_graphs, server_url)
iterator, query_vars, c = build_left_plan(triples, dataset, current_graphs)
# track cardinalities of every triple pattern
cardinalities += c
return iterator
else:
raise UnsupportedSPARQL("Unsupported SPARQL feature: {}".format(node.name))
def build_query_plan(query, dataset, default_graph, saved_plan=None):
"""Build a pipeline of iterators used to evaluate a query"""
cardinalities = []
if saved_plan is not None:
return load(saved_plan, dataset), []
root = None
if query['type'] == 'union':
root, cardinalities = build_union_plan(query['union'], dataset, default_graph)
elif query['type'] == 'bgp':
root, cardinalities = build_join_plan(query['bgp'], dataset, default_graph)
else:
raise Exception('Unkown query type found during query optimization')
# apply (possible) filter clause(s)
if 'filters' in query and len(query['filters']) > 0:
# exclude empty strings
filters = list(filter(lambda x: len(x) > 0, query['filters']))
if len(filters) > 0:
# reduce all filters in a conjunctive expression
expression = reduce(lambda x, y: "({}) && ({})".format(x, y), filters)
root = FilterIterator(root, expression)
return root, cardinalities
def load_filter(saved_plan, dataset):
"""Load a FilterIterator from a protobuf serialization"""
sourceField = saved_plan.WhichOneof('source')
source = load(getattr(saved_plan, sourceField), dataset)
return FilterIterator(source, saved_plan.expression)