def load(protoMsg, dataset):
"""Load a preemptable physical query execution plan from a saved state"""
saved_plan = protoMsg
if isinstance(protoMsg, bytes):
root = RootTree()
root.ParseFromString(protoMsg)
sourceField = root.WhichOneof('source')
saved_plan = getattr(root, sourceField)
if type(saved_plan) is SavedFilterIterator:
return load_filter(saved_plan, dataset)
if type(saved_plan) is SavedProjectionIterator:
return load_projection(saved_plan, dataset)
elif type(saved_plan) is SavedScanIterator:
return load_scan(saved_plan, dataset)
elif type(saved_plan) is SavedIndexJoinIterator:
return load_nlj(saved_plan, dataset)
elif type(saved_plan) is SavedBagUnionIterator:
return load_union(saved_plan, dataset)
elif type(saved_plan) is SavedGroupByAgg:
return load_groupby(saved_plan, dataset)
elif type(saved_plan) is SavedAggregatesProjectionIterator:
return load_aggregates_projection(saved_plan, dataset)
else:
raise Exception('Unknown iterator type "%s" when loading controls' %
type(saved_plan))
def load(saved_plan: SavedProtobufPlan, dataset: Dataset, context: dict) -> PreemptableIterator:
"""Load a preemptable physical query execution plan from a saved state.
Args:
* saved_plan: Saved query execution plan.
* dataset: RDF dataset used to execute the plan.
* context: Information about the query execution.
Returns:
The pipeline of iterator used to continue query execution.
"""
# unpack the plan from the serialized protobuf message
if isinstance(saved_plan, bytes):
root = RootTree()
root.ParseFromString(saved_plan)
sourceField = root.WhichOneof('source')
saved_plan = getattr(root, sourceField)
# load the plan based on the current node
if type(saved_plan) is SavedFilterIterator:
return load_filter(saved_plan, dataset, context)
if type(saved_plan) is SavedProjectionIterator:
return load_projection(saved_plan, dataset, context)
elif type(saved_plan) is SavedScanIterator:
return load_scan(saved_plan, dataset, context)
elif type(saved_plan) is SavedIndexJoinIterator:
return load_nlj(saved_plan, dataset, context)
elif type(saved_plan) is SavedBagUnionIterator:
return load_union(saved_plan, dataset, context)
else:
raise Exception(f"Unknown iterator type '{type(saved_plan)}' when loading controls")
def load(saved_plan: SavedProtobufPlan, dataset: Dataset) -> PreemptableIterator:
"""Load a preemptable physical query execution plan from a saved state.
Args:
* saved_plan: Saved query execution plan.
* dataset: RDF dataset used to execute the plan.
Returns:
The pipeline of iterator used to continue query execution.
"""
# unpack the plan from the serialized protobuf message
try:
# print(f"...{type(saved_plan)}...")
if isinstance(saved_plan, bytes):
root = RootTree()
root.ParseFromString(saved_plan)
sourceField = root.WhichOneof('source')
saved_plan = getattr(root, sourceField)
# load the plan based on the current node
if type(saved_plan) is SavedFilterIterator:
return load_filter(saved_plan, dataset)
elif type(saved_plan) is SavedProjectionIterator:
return load_projection(saved_plan, dataset)
elif type(saved_plan) is SavedReducedIterator:
return load_reduced(saved_plan, dataset)
elif type(saved_plan) is SavedScanIterator:
return load_scan(saved_plan, dataset)
elif type(saved_plan) is SavedIndexJoinIterator:
return load_nlj(saved_plan, dataset)
elif type(saved_plan) is SavedBagUnionIterator:
return load_union(saved_plan, dataset)
elif type(saved_plan) is SavedBindIterator:
return load_bind(saved_plan, dataset)
elif type(saved_plan) is SavedConstructIterator:
return load_construct(saved_plan, dataset)
else:
raise Exception(f"Unknown iterator type '{type(saved_plan)}' when loading controls")
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
logging.error(f"load_plan:{sys.exc_info()[0]}")
raise
def sage_query(config_file, default_graph_uri, query, file, limit):
"""
Execute a SPARQL query on an embedded Sage Server.
Example usage: sage-query config.yaml http://example.org/swdf-postgres -f queries/spo.sparql
"""
# assert that we have a query to evaluate
if query is None and file is None:
print("Error: you must specificy a query to execute, either with --query or --file. See sage-query --help for more informations.")
exit(1)
if limit is None:
limit = inf
# load query from file if required
if file is not None:
with open(file) as query_file:
query = query_file.read()
# dataset = load_config(config_file)
# if not dataset.has_graph(default_graph_uri):
# print("Error: the config_file does not define your {default_graph_uri}.")
client=TestClient(run_app(config_file))
nbResults = 0
nbCalls = 0
hasNext = True
next_link = None
count=0
start = time()
while hasNext:
response = post_sparql(client, query, next_link, default_graph_uri)
response = response.json()
nbResults += len(response['bindings'])
hasNext = response['hasNext']
next_link = response['next']
nbCalls += 1
for bindings in response['bindings']:
print(bindings)
# for k,v in bindings.items():
# print(f"{v} ")
if next_link is not None:
saved_plan = next_link
plan = decode_saved_plan(saved_plan)
root = RootTree()
root.ParseFromString(plan)
prog,card=progress(root)
logger.info(f"progression {prog}/{card}:{prog/card*100}%")
count += 1
if count >= limit:
break
end= time()
logger.info("finished in {}s".format(end-start))
logger.info("made {} calls".format(nbCalls))
logger.info("got {} mappings".format(nbResults))