def pythonmain(): testexpr1 = Join([("object","subject")], Scan("R"), Scan("R")) R = Scan(Rr) #sR = Select("lambda t: t.predicate == 'knows'", R) #sS = Select("lambda t: t.predicate == 'holdsAccount'", R) #sT = Select("lambda t: t.predicate == 'accountServiceHomepage'", R) sR = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("knows")), R) sS = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount")), R) sT = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("accountServiceHomepage")), R) sRsS = Join([("object","subject")], sR, sS) sRsSsT = Join([("object1", "subject")], sRsS, sT) ssR = Select(EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount")), sR) test = Select(OR(EQ(NamedAttributeRef("predicate"), StringLiteral("knows")),EQ(NamedAttributeRef("predicate"), StringLiteral("holdsAccount"))), R) #print ssR ossR = optimize(test, target=PythonAlgebra, source=LogicalAlgebra) #ossR = optimize(ssR, target=PseudoCodeAlgebra, source=LogicalAlgebra) #print ossR #print compile(ossR) #print compile(sRsSsT) #print [gensym() for i in range(5)] #sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(330337405)), R) osR = optimize(sRsSsT, target=PythonAlgebra, source=LogicalAlgebra) print compile(osR)
def ccmain(): # c implementation doesn't support string literals R = Scan(btc_schema["trial"]) sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(1133564893)), R) sS = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R) sT = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R) sRsS = Join([("object","subject")], sR, sS) sRsSsT = Join([("object","subject")], sRsS, sT) """ """ result = optimize([('ans', sRsSsT)], target=CCAlgebra, source=LogicalAlgebra) #result = optimize(sR, target=CCAlgebra, source=LogicalAlgebra) return compile(result) def f(op): yield op #.__class__.__name__ def g(op, vars): if op in vars: del vars[op] yield (op,vars.copy()) def show(op): yield op.test #vars = dict([(x,1) for x in sR.postorder(f)]) #vars.pop(sR, None) parents = {sRsSsT:[None]} sRsSsT.collectParents(parents)
def check(self, query, name, **kwargs): gname = "grappa_{name}".format(name=name) if kwargs.get('join_type', None) == 'symmetric_hash': kwargs['join_type'] = grappalang.GrappaSymmetricHashJoin elif kwargs.get('join_type', None) == 'shuffle_hash': kwargs['join_type'] = grappalang.GrappaShuffleHashJoin # FIXME: see issue #348; always skipping shuffle tests because it got broken raise SkipTest() kwargs['target_alg'] = GrappaAlgebra() plan = self.get_physical_plan(query, **kwargs) physical_dot = viz.operator_to_dot(plan) with open(os.path.join("c_test_environment", "{gname}.physical.dot".format(gname=gname)), 'w') as dwf: dwf.write(physical_dot) # generate code in the target language code = compile(plan) fname = os.path.join("c_test_environment", "{gname}.cpp".format(gname=gname)) if os.path.exists(fname): os.remove(fname) with open(fname, 'w') as f: f.write(code) raise_skip_test(query) with Chdir("c_test_environment") as d: checkquery(name, GrappalangRunner())
def check(self, query, name, join_type=None, emit_print='console', **kwargs): gname = "grappa_{name}".format(name=name) plan = self._get_grappa_physical_plan(query, join_type, emit_print, **kwargs) physical_dot = viz.operator_to_dot(plan) with open(os.path.join("c_test_environment", "{gname}.physical.dot".format(gname=gname)), 'w') as dwf: dwf.write(physical_dot) # generate code in the target language # test_mode=True turns on extra checks like assign-once instance # variables for operators code = compile(plan, test_mode=True, **kwargs) fname = os.path.join("c_test_environment", "{gname}.cpp".format(gname=gname)) if os.path.exists(fname): os.remove(fname) with open(fname, 'w') as f: f.write(code) #raise Exception() raise_skip_test(query) with Chdir("c_test_environment") as d: if emit_print == 'file': checkstore(name, GrappalangRunner(binary_input=False)) else: checkquery(name, GrappalangRunner(binary_input=False))
def emitCode(query, name, algebra): LOG.info("compiling %s: %s", name, query) # Create a compiler object dlog = RACompiler() # parse the query dlog.fromDatalog(query) #print dlog.parsed LOG.info("logical: %s",dlog.logicalplan) generateDot.generateDot(dlog.logicalplan, "%s.logical.dot"%(name)) dlog.optimize(target=algebra, eliminate_common_subexpressions=False) LOG.info("physical: %s",dlog.physicalplan[0][1]) generateDot.generateDot(dlog.physicalplan, "%s.physical.dot"%(name)) # generate code in the target language code = "" code += comment("Query " + query) code += compile(dlog.physicalplan) fname = name+'.cpp' with open(fname, 'w') as f: f.write(code) # returns name of code file return fname
def check(self, query, name, **kwargs): with Chdir("c_test_environment") as d: kwargs['target_alg'] = CCAlgebra() plan = self.get_physical_plan(query, **kwargs) physical_dot = viz.operator_to_dot(plan) with open("%s.physical.dot"%(name), 'w') as dwf: dwf.write(physical_dot) # generate code in the target language code = compile(plan) fname = "{name}.cpp".format(name=name) if os.path.exists(fname): os.remove(fname) with open(fname, 'w') as f: f.write(code) checkquery(name, ClangRunner())
def emitCode(query, name, algType, plan=None, emit_print=None, dir='.'): if emit_print is not None: alg = algType(emit_print) else: alg = algType() hack_plan(alg, plan) LOG.info("compiling %s: %s", name, query) # Create a compiler object dlog = RACompiler() # parse the query dlog.fromDatalog(query) # print dlog.parsed LOG.info("logical: %s", dlog.logicalplan) print dlog.logicalplan logical_dot = viz.operator_to_dot(dlog.logicalplan) with open("%s.logical.dot" % (name), 'w') as dwf: dwf.write(logical_dot) dlog.optimize(target=alg) LOG.info("physical: %s", dlog.physicalplan) print dlog.physicalplan physical_dot = viz.operator_to_dot(dlog.physicalplan) with open("%s.physical.dot" % (name), 'w') as dwf: dwf.write(physical_dot) # generate code in the target language code = "" code += comment("Query " + query) code += compile(dlog.physicalplan) fname = '{dir}/{name}.cpp'.format(dir=dir, name=name) with open(fname, 'w') as f: f.write(code) # returns name of code file return fname
def check(self, query, name, **kwargs): kwargs['target_alg'] = CCAlgebra() plan = self.get_physical_plan(query, **kwargs) physical_dot = viz.operator_to_dot(plan) with open( os.path.join("c_test_environment", "%s.physical.dot" % (name)), 'w') as dwf: dwf.write(physical_dot) # generate code in the target language code = compile(plan) fname = os.path.join("c_test_environment", "{name}.cpp".format(name=name)) if os.path.exists(fname): os.remove(fname) with open(fname, 'w') as f: f.write(code) with Chdir("c_test_environment") as d: checkquery(name, ClangRunner())
#ra = onlyrule.toRA(parsedprogram) #print exprs ra = exprs print query print ra print "//-------" # optimize applies a set of rules to translate a source # expression to a target expression #result = optimize(ra, target=PseudoCodeAlgebra, source=LogicalAlgebra) #result = optimize(ra, target=ProtobufAlgebra, source=LogicalAlgebra) result = optimize(ra, target=MyriaAlgebra, source=LogicalAlgebra) print result #comment(result) physicalplan = result #physicalplan = common_subexpression_elimination(result) #comment(physicalplan) #for x in showids(physicalplan): # print x # generate code in the target language #print compile(physicalplan) #compile(physicalplan) print compile(physicalplan)
print '%s%s' % (' ' * indent, plan.shortStr()) for child in plan.children(): print_pretty_plan(child, indent + 4) else: print '%s%s' % (' ' * indent, plan) catalog = FromFileCatalog.load_from_file("catalog.py") _parser = parser.Parser() query = "" with open(sys.argv[1], 'r') as f: query = f.read() statement_list = _parser.parse(query) processor = interpreter.StatementProcessor(catalog, True) processor.evaluate(statement_list) # we will add the shuffle into the logical plan print "LOGICAL" p = processor.get_logical_plan() print_pretty_plan(p) print "PHYSICAL" #p = processor.get_physical_plan(push_sql=True) p = processor.get_physical_plan(target_alg=GrappaAlgebra(), groupby_semantics='partition') print_pretty_plan(p) print p with open('{}.cpp'.format(sys.argv[1]), 'w') as f: f.write(compile(p))
sch = raco.scheme.Scheme([("subject", int), ("predicate", int), ("object", int)]) # Create a relation object. We can add formats here as needed. trialdat = raco.catalog.ASCIIFile("trial.dat", sch) print sch # Now write the RA expression # Scan just takes a pointer to a relation object R = Scan(trialdat, sch) #TODO: is this supposed to pass sch? print R.scheme() # Select # EQ(x,y) means x=y, GT(x,y) means x>y, etc. sR = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(1133564893)), R) sS = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R) #sT = Select(EQ(NamedAttributeRef("predicate"), NumericLiteral(77645021)), R) sT = Select(EQ(NamedAttributeRef("object"), NumericLiteral(1018848684)), R) # Join([(w,x),(y,z)], R, S) means "JOIN R, S ON (R.w = S.x AND R.y = S.z)" sRsS = Join([("object","subject")], sR, sS) sRsSsT = Join([("object","subject")], sRsS, sT) # optimize applies a set of rules to translate a source # expression to a target expression result = optimize([("Ans", sT)], CCAlgebra) # compile generates the linear code from the expression tree print compile(result)