def execute_query(self, query):
'''Run a test query against the fake database'''
# print query
dlog = RACompiler()
dlog.fromDatalog(query)
# print dlog.logicalplan
dlog.optimize(target=MyriaAlgebra,
eliminate_common_subexpressions=False)
# print dlog.physicalplan
# test whether we can generate json without errors
json_string = json.dumps(compile_to_json(
query, dlog.logicalplan, dlog.physicalplan))
assert json_string
op = dlog.physicalplan[0][1]
output_op = raco.algebra.Store(RelationKey.from_string('__OUTPUT__'),
op)
self.db.evaluate(output_op)
return self.db.get_table('__OUTPUT__')
def emitCode(query, name, algebra):
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
#print dlog.parsed
LOG.info("logical: %s",dlog.logicalplan)
generateDot.generateDot(dlog.logicalplan, "%s.logical.dot"%(name))
dlog.optimize(target=algebra, eliminate_common_subexpressions=False)
LOG.info("physical: %s",dlog.physicalplan[0][1])
generateDot.generateDot(dlog.physicalplan, "%s.physical.dot"%(name))
# generate code in the target language
code = ""
code += comment("Query " + query)
code += compile(dlog.physicalplan)
fname = name+'.cpp'
with open(fname, 'w') as f:
f.write(code)
# returns name of code file
return fname
def main():
# A simple join
query = "A(x,z) :- R(x,y), S(y,z)"
# Triangle
# query = "A(x,z) :- R(x,y),S(y,z),T(z,x)"
# Two independent rules
query = """A(x,z) :- R(x,y), S(y,z); B(x,z) :- S(z,x)."""
# Create a cmpiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
# Print out the graph
for (label, root_operator) in dlog.logicalplan:
graph = root_operator.collectGraph()
v1 = graph_to_dot(graph)
v2 = operator_to_dot(root_operator)
assert v1 == v2
# print "Dot for individual IDB %s: " % label
# print v1
# print
v3 = plan_to_dot(dlog.logicalplan)
# print "Dot for combined IDBs:"
print v3
# print
if len(dlog.logicalplan) == 1:
assert v1 == v3
def get_phys_plan_root(query, num_server, child_size=None):
dlog = RACompiler()
dlog.fromDatalog(query)
dlog.optimize(myrialang.MyriaHyperCubeAlgebra(
FakeCatalog(num_server, child_size)))
ret = dlog.physicalplan
assert isinstance(ret, algebra.Parallel)
assert len(ret.children()) == 1
ret = ret.children()[0]
assert isinstance(ret, algebra.Store)
return ret.input
def execute_query(self,
query,
test_logical=False,
skip_json=False,
output="OUTPUT",
algebra=MyriaLeftDeepTreeAlgebra):
"""Run a test query against the fake database"""
dlog = RACompiler()
dlog.fromDatalog(query)
assert algebra in [MyriaLeftDeepTreeAlgebra, MyriaHyperCubeAlgebra]
if test_logical:
plan = dlog.logicalplan
else:
if algebra == MyriaLeftDeepTreeAlgebra:
dlog.optimize(MyriaLeftDeepTreeAlgebra())
else:
dlog.optimize(MyriaHyperCubeAlgebra(FakeCatalog(64)))
plan = dlog.physicalplan
if not skip_json:
# test whether we can generate json without errors
json_string = json.dumps(
compile_to_json(query, dlog.logicalplan, dlog.physicalplan,
"datalog"))
assert json_string
self.db.evaluate(plan)
return self.db.get_table(output)
def _get_datalog_plan(query, plan_type, algebra, **kwargs):
datalog = RACompiler()
datalog.fromDatalog(query)
if not datalog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
elif plan_type == 'logical':
return datalog.logicalplan
elif plan_type == 'physical':
datalog.optimize(target=algebra,
push_sql=kwargs.get('push_sql', True))
return datalog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
def get_plan(query, language, plan_type, connection,
multiway_join=False, push_sql=False):
catalog = None
if multiway_join:
catalog = MyriaCatalog(connection)
assert catalog.get_num_servers()
# Fix up the language string
if language is None:
language = "datalog"
language = language.strip().lower()
if multiway_join:
target_algebra = MyriaHyperCubeAlgebra(catalog)
else:
target_algebra = MyriaLeftDeepTreeAlgebra()
if language == "datalog":
dlog = RACompiler()
dlog.fromDatalog(query)
if not dlog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
if plan_type == 'logical':
return dlog.logicalplan
elif plan_type == 'physical':
dlog.optimize(target=target_algebra, push_sql=push_sql)
return dlog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
elif language in ["myrial", "sql"]:
# We need a (global) lock on the Myrial parser because yacc
# .. is not Threadsafe and App Engine uses multiple threads.
with myrial_parser_lock:
parsed = myrial_parser.parse(query)
processor = MyrialInterpreter.StatementProcessor(
MyriaCatalog(connection))
processor.evaluate(parsed)
if plan_type == 'logical':
return processor.get_physical_plan(target_alg=OptLogicalAlgebra())
elif plan_type == 'physical':
return processor.get_physical_plan(target_alg=target_algebra,
multiway_join=multiway_join,
push_sql=push_sql)
else:
raise NotImplementedError('Myria plan type %s' % plan_type)
raise NotImplementedError('Language %s is not supported' % language)
def execute_query(self, query, test_logical=False, skip_json=False,
output="OUTPUT", algebra=MyriaLeftDeepTreeAlgebra):
"""Run a test query against the fake database"""
dlog = RACompiler()
dlog.fromDatalog(query)
assert algebra in [MyriaLeftDeepTreeAlgebra,
MyriaHyperCubeAlgebra]
if test_logical:
plan = dlog.logicalplan
else:
if algebra == MyriaLeftDeepTreeAlgebra:
dlog.optimize(MyriaLeftDeepTreeAlgebra())
else:
dlog.optimize(MyriaHyperCubeAlgebra(FakeCatalog(64)))
plan = dlog.physicalplan
if not skip_json:
# test whether we can generate json without errors
json_string = json.dumps(compile_to_json(
query, dlog.logicalplan, dlog.physicalplan, "datalog"))
assert json_string
self.db.evaluate(plan)
return self.db.get_table(output)
def get_plan(query, language, plan_type, connection):
# Fix up the language string
if language is None:
language = "datalog"
language = language.strip().lower()
if language == "datalog":
dlog = RACompiler()
dlog.fromDatalog(query)
if not dlog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
if plan_type == 'logical':
return dlog.logicalplan
dlog.optimize(target=MyriaAlgebra,
eliminate_common_subexpressions=False)
if plan_type == 'physical':
return dlog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
elif language in ["myrial", "sql"]:
# We need a (global) lock on the Myrial parser because yacc is not
# ... Threadsafe and App Engine uses multiple threads.
with myrial_parser_lock:
parsed = myrial_parser.parse(query)
processor = MyrialInterpreter.StatementProcessor(
MyriaCatalog(connection))
processor.evaluate(parsed)
if plan_type == 'logical':
return processor.get_logical_plan()
elif plan_type == 'physical':
return processor.get_physical_plan()
else:
raise NotImplementedError('Myria plan type %s' % plan_type)
else:
raise NotImplementedError('Language %s is not supported' % language)
raise NotImplementedError('Should not be able to get here')
def get_phys_plan_root(query, num_server, child_size=None):
dlog = RACompiler()
dlog.fromDatalog(query)
dlog.optimize(
myrialang.MyriaHyperCubeAlgebra(FakeCatalog(
num_server, child_size)))
ret = dlog.physicalplan
assert isinstance(ret, algebra.Parallel)
assert len(ret.children()) == 1
ret = ret.children()[0]
assert isinstance(ret, algebra.Store)
return ret.input
def get_plan(query,
language,
plan_type,
connection,
multiway_join=False,
push_sql=False):
catalog = None
if multiway_join:
catalog = MyriaCatalog(connection)
assert catalog.get_num_servers()
# Fix up the language string
if language is None:
language = "datalog"
language = language.strip().lower()
if multiway_join:
target_algebra = MyriaHyperCubeAlgebra(catalog)
else:
target_algebra = MyriaLeftDeepTreeAlgebra()
if language == "datalog":
dlog = RACompiler()
dlog.fromDatalog(query)
if not dlog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
if plan_type == 'logical':
return dlog.logicalplan
elif plan_type == 'physical':
dlog.optimize(target=target_algebra, push_sql=push_sql)
return dlog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
elif language in ["myrial", "sql"]:
# We need a (global) lock on the Myrial parser because yacc
# .. is not Threadsafe and App Engine uses multiple threads.
with myrial_parser_lock:
parsed = myrial_parser.parse(query)
processor = MyrialInterpreter.StatementProcessor(
MyriaCatalog(connection))
processor.evaluate(parsed)
if plan_type == 'logical':
return processor.get_physical_plan(target_alg=OptLogicalAlgebra())
elif plan_type == 'physical':
return processor.get_physical_plan(target_alg=target_algebra,
multiway_join=multiway_join,
push_sql=push_sql)
else:
raise NotImplementedError('Myria plan type %s' % plan_type)
raise NotImplementedError('Language %s is not supported' % language)
def _get_datalog_plan(query, plan_type, algebra, **kwargs):
datalog = RACompiler()
datalog.fromDatalog(query)
if not datalog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
elif plan_type == 'logical':
return datalog.logicalplan
elif plan_type == 'physical':
datalog.optimize(target=algebra,
push_sql=kwargs.get('push_sql', True))
return datalog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
def testEmit(query, name):
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
#print dlog.parsed
LOG.info("logical: %s", dlog.logicalplan)
dlog.optimize(target=GrappaAlgebra)
LOG.info("physical: %s", dlog.physicalplan[0][1])
# generate code in the target language
code = ""
code += comment("Query " + query)
code += dlog.compile()
with open(name + '.cpp', 'w') as f:
f.write(code)
def emitCode(query, name, algType, plan=None, emit_print=None, dir='.'):
if emit_print is not None:
alg = algType(emit_print)
else:
alg = algType()
hack_plan(alg, plan)
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
# print dlog.parsed
LOG.info("logical: %s", dlog.logicalplan)
print dlog.logicalplan
logical_dot = viz.operator_to_dot(dlog.logicalplan)
with open("%s.logical.dot" % (name), 'w') as dwf:
dwf.write(logical_dot)
dlog.optimize(target=alg)
LOG.info("physical: %s", dlog.physicalplan)
print dlog.physicalplan
physical_dot = viz.operator_to_dot(dlog.physicalplan)
with open("%s.physical.dot" % (name), 'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
code = ""
code += comment("Query " + query)
code += compile(dlog.physicalplan)
fname = '{dir}/{name}.cpp'.format(dir=dir, name=name)
with open(fname, 'w') as f:
f.write(code)
# returns name of code file
return fname
def testEmit(query, name):
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
#print dlog.parsed
LOG.info("logical: %s",dlog.logicalplan)
dlog.optimize(target=GrappaAlgebra)
LOG.info("physical: %s",dlog.physicalplan[0][1])
# generate code in the target language
code = ""
code += comment("Query " + query)
code += dlog.compile()
with open(name+'.cpp', 'w') as f:
f.write(code)
def get_plan(query, language, plan_type, connection):
# Fix up the language string
if language is None:
language = "datalog"
language = language.strip().lower()
if language == "datalog":
dlog = RACompiler()
dlog.fromDatalog(query)
if not dlog.logicalplan:
raise SyntaxError("Unable to parse Datalog")
if plan_type == 'logical':
return dlog.logicalplan
dlog.optimize(target=MyriaAlgebra,
eliminate_common_subexpressions=False)
if plan_type == 'physical':
return dlog.physicalplan
else:
raise NotImplementedError('Datalog plan type %s' % plan_type)
elif language in ["myrial", "sql"]:
# We need a (global) lock on the Myrial parser because yacc is not
# ... Threadsafe and App Engine uses multiple threads.
with myrial_parser_lock:
parsed = myrial_parser.parse(query)
processor = MyrialInterpreter.StatementProcessor(
MyriaCatalog(connection))
processor.evaluate(parsed)
if plan_type == 'logical':
return processor.get_logical_plan()
elif plan_type == 'physical':
return processor.get_physical_plan()
else:
raise NotImplementedError('Myria plan type %s' % plan_type)
else:
raise NotImplementedError('Language %s is not supported' % language)
raise NotImplementedError('Should not be able to get here')
# Compute the cluster centers. First, compute the server-local cluster centers CL(cid, avg(x) ) :- C(pid, cid), P(pid, x) """ query = """ A@*(x) :- R(x,y,z) """ query = """ smallTableJoin(x,z) :- smallTable(x,y),smallTable(y,z) """ def comment(s): print "/*\n%s\n*/" % str(s) dlog = RACompiler() dlog.fromDatalog(query) print dlog.logicalplan dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False) code = dlog.compile() print code # generate code in the target language #print compile(physicalplan) #compile(physicalplan) print compile(physicalplan)
def RATest(query):
dlog = RACompiler()
dlog.fromDatalog(query)
# TODO: Testing for string equality, but we should do something like what
# Andrew does -- evaluate the expressions on test data.
return "%s" % dlog.logicalplan