def pythonmain():
testexpr1 = Join([("object","subject")], Scan("R"), Scan("R"))
R = Scan(Rr)
#sR = Select("lambda t: t.predicate == 'knows'", R)
#sS = Select("lambda t: t.predicate == 'holdsAccount'", R)
#sT = Select("lambda t: t.predicate == 'accountServiceHomepage'", R)
sR = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("knows")), R)
sS = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount")), R)
sT = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("accountServiceHomepage")), R)
sRsS = Join([("object","subject")], sR, sS)
sRsSsT = Join([("object1", "subject")], sRsS, sT)
ssR = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount")), sR)
test = Select(OR(EQ(NamedAttributeRef("predicate"), StringLiteral("knows")),EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount"))), R)
#print ssR
ossR = optimize(test, target=PythonAlgebra, source=LogicalAlgebra)
#ossR = optimize(ssR, target=PseudoCodeAlgebra, source=LogicalAlgebra)
#print ossR
#print compile(ossR)
#print compile(sRsSsT)
#print [gensym() for i in range(5)]
#sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(330337405)), R)
osR = optimize(sRsSsT, target=PythonAlgebra, source=LogicalAlgebra)
print compile(osR)
def ccmain():
# c implementation doesn't support string literals
R = Scan(btc_schema["trial"])
sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(1133564893)), R)
sS = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R)
sT = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R)
sRsS = Join([("object","subject")], sR, sS)
sRsSsT = Join([("object","subject")], sRsS, sT)
"""
"""
result = optimize([('ans', sRsSsT)], target=CCAlgebra, source=LogicalAlgebra)
#result = optimize(sR, target=CCAlgebra, source=LogicalAlgebra)
return compile(result)
def f(op):
yield op #.__class__.__name__
def g(op, vars):
if op in vars: del vars[op]
yield (op,vars.copy())
def show(op):
yield op.test
#vars = dict([(x,1) for x in sR.postorder(f)])
#vars.pop(sR, None)
parents = {sRsSsT:[None]}
sRsSsT.collectParents(parents)
def check(self, query, name, **kwargs):
gname = "grappa_{name}".format(name=name)
if kwargs.get('join_type', None) == 'symmetric_hash':
kwargs['join_type'] = grappalang.GrappaSymmetricHashJoin
elif kwargs.get('join_type', None) == 'shuffle_hash':
kwargs['join_type'] = grappalang.GrappaShuffleHashJoin
# FIXME: see issue #348; always skipping shuffle tests because it got broken
raise SkipTest()
kwargs['target_alg'] = GrappaAlgebra()
plan = self.get_physical_plan(query, **kwargs)
physical_dot = viz.operator_to_dot(plan)
with open(os.path.join("c_test_environment",
"{gname}.physical.dot".format(gname=gname)), 'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
code = compile(plan)
fname = os.path.join("c_test_environment", "{gname}.cpp".format(gname=gname))
if os.path.exists(fname):
os.remove(fname)
with open(fname, 'w') as f:
f.write(code)
raise_skip_test(query)
with Chdir("c_test_environment") as d:
checkquery(name, GrappalangRunner())
def check(self, query, name, join_type=None, emit_print='console', **kwargs):
gname = "grappa_{name}".format(name=name)
plan = self._get_grappa_physical_plan(query, join_type, emit_print, **kwargs)
physical_dot = viz.operator_to_dot(plan)
with open(os.path.join("c_test_environment",
"{gname}.physical.dot".format(gname=gname)), 'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
# test_mode=True turns on extra checks like assign-once instance
# variables for operators
code = compile(plan, test_mode=True, **kwargs)
fname = os.path.join("c_test_environment", "{gname}.cpp".format(gname=gname))
if os.path.exists(fname):
os.remove(fname)
with open(fname, 'w') as f:
f.write(code)
#raise Exception()
raise_skip_test(query)
with Chdir("c_test_environment") as d:
if emit_print == 'file':
checkstore(name, GrappalangRunner(binary_input=False))
else:
checkquery(name, GrappalangRunner(binary_input=False))
def emitCode(query, name, algebra):
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
#print dlog.parsed
LOG.info("logical: %s",dlog.logicalplan)
generateDot.generateDot(dlog.logicalplan, "%s.logical.dot"%(name))
dlog.optimize(target=algebra, eliminate_common_subexpressions=False)
LOG.info("physical: %s",dlog.physicalplan[0][1])
generateDot.generateDot(dlog.physicalplan, "%s.physical.dot"%(name))
# generate code in the target language
code = ""
code += comment("Query " + query)
code += compile(dlog.physicalplan)
fname = name+'.cpp'
with open(fname, 'w') as f:
f.write(code)
# returns name of code file
return fname
def check(self, query, name, **kwargs):
with Chdir("c_test_environment") as d:
kwargs['target_alg'] = CCAlgebra()
plan = self.get_physical_plan(query, **kwargs)
physical_dot = viz.operator_to_dot(plan)
with open("%s.physical.dot"%(name), 'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
code = compile(plan)
fname = "{name}.cpp".format(name=name)
if os.path.exists(fname):
os.remove(fname)
with open(fname, 'w') as f:
f.write(code)
checkquery(name, ClangRunner())
def emitCode(query, name, algType, plan=None, emit_print=None, dir='.'):
if emit_print is not None:
alg = algType(emit_print)
else:
alg = algType()
hack_plan(alg, plan)
LOG.info("compiling %s: %s", name, query)
# Create a compiler object
dlog = RACompiler()
# parse the query
dlog.fromDatalog(query)
# print dlog.parsed
LOG.info("logical: %s", dlog.logicalplan)
print dlog.logicalplan
logical_dot = viz.operator_to_dot(dlog.logicalplan)
with open("%s.logical.dot" % (name), 'w') as dwf:
dwf.write(logical_dot)
dlog.optimize(target=alg)
LOG.info("physical: %s", dlog.physicalplan)
print dlog.physicalplan
physical_dot = viz.operator_to_dot(dlog.physicalplan)
with open("%s.physical.dot" % (name), 'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
code = ""
code += comment("Query " + query)
code += compile(dlog.physicalplan)
fname = '{dir}/{name}.cpp'.format(dir=dir, name=name)
with open(fname, 'w') as f:
f.write(code)
# returns name of code file
return fname
def check(self, query, name, **kwargs):
kwargs['target_alg'] = CCAlgebra()
plan = self.get_physical_plan(query, **kwargs)
physical_dot = viz.operator_to_dot(plan)
with open(
os.path.join("c_test_environment", "%s.physical.dot" % (name)),
'w') as dwf:
dwf.write(physical_dot)
# generate code in the target language
code = compile(plan)
fname = os.path.join("c_test_environment",
"{name}.cpp".format(name=name))
if os.path.exists(fname):
os.remove(fname)
with open(fname, 'w') as f:
f.write(code)
with Chdir("c_test_environment") as d:
checkquery(name, ClangRunner())
#ra = onlyrule.toRA(parsedprogram) #print exprs ra = exprs print query print ra print "//-------" # optimize applies a set of rules to translate a source # expression to a target expression #result = optimize(ra, target=PseudoCodeAlgebra, source=LogicalAlgebra) #result = optimize(ra, target=ProtobufAlgebra, source=LogicalAlgebra) result = optimize(ra, target=MyriaAlgebra, source=LogicalAlgebra) print result #comment(result) physicalplan = result #physicalplan = common_subexpression_elimination(result) #comment(physicalplan) #for x in showids(physicalplan): # print x # generate code in the target language #print compile(physicalplan) #compile(physicalplan) print compile(physicalplan)
print '%s%s' % (' ' * indent, plan.shortStr())
for child in plan.children():
print_pretty_plan(child, indent + 4)
else:
print '%s%s' % (' ' * indent, plan)
catalog = FromFileCatalog.load_from_file("catalog.py")
_parser = parser.Parser()
query = ""
with open(sys.argv[1], 'r') as f:
query = f.read()
statement_list = _parser.parse(query)
processor = interpreter.StatementProcessor(catalog, True)
processor.evaluate(statement_list)
# we will add the shuffle into the logical plan
print "LOGICAL"
p = processor.get_logical_plan()
print_pretty_plan(p)
print "PHYSICAL"
#p = processor.get_physical_plan(push_sql=True)
p = processor.get_physical_plan(target_alg=GrappaAlgebra(), groupby_semantics='partition')
print_pretty_plan(p)
print p
with open('{}.cpp'.format(sys.argv[1]), 'w') as f:
f.write(compile(p))
sch = raco.scheme.Scheme([("subject", int), ("predicate", int), ("object", int)])
# Create a relation object. We can add formats here as needed.
trialdat = raco.catalog.ASCIIFile("trial.dat", sch)
print sch
# Now write the RA expression
# Scan just takes a pointer to a relation object
R = Scan(trialdat, sch) #TODO: is this supposed to pass sch?
print R.scheme()
# Select
# EQ(x,y) means x=y, GT(x,y) means x>y, etc.
sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(1133564893)), R)
sS = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R)
#sT = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R)
sT = Select(EQ(NamedAttributeRef("object"), NumericLiteral(1018848684)), R)
# Join([(w,x),(y,z)], R, S) means "JOIN R, S ON (R.w = S.x AND R.y = S.z)"
sRsS = Join([("object","subject")], sR, sS)
sRsSsT = Join([("object","subject")], sRsS, sT)
# optimize applies a set of rules to translate a source
# expression to a target expression
result = optimize([("Ans", sT)], CCAlgebra)
# compile generates the linear code from the expression tree
print compile(result)
