Exemple #1
0
    def execute_query(self, query):
        '''Run a test query against the fake database'''

        # print query

        dlog = RACompiler()
        dlog.fromDatalog(query)

        # print dlog.logicalplan

        dlog.optimize(target=MyriaAlgebra,
                      eliminate_common_subexpressions=False)

        # print dlog.physicalplan

        # test whether we can generate json without errors
        json_string = json.dumps(compile_to_json(
            query, dlog.logicalplan, dlog.physicalplan))
        assert json_string

        op = dlog.physicalplan[0][1]
        output_op = raco.algebra.Store(RelationKey.from_string('__OUTPUT__'),
                                       op)
        self.db.evaluate(output_op)
        return self.db.get_table('__OUTPUT__')
Exemple #2
0
def emitCode(query, name, algebra):
    LOG.info("compiling %s: %s", name, query)

    # Create a compiler object
    dlog = RACompiler()

    # parse the query
    dlog.fromDatalog(query)
    #print dlog.parsed
    LOG.info("logical: %s",dlog.logicalplan)

    generateDot.generateDot(dlog.logicalplan, "%s.logical.dot"%(name))

    dlog.optimize(target=algebra, eliminate_common_subexpressions=False)

    LOG.info("physical: %s",dlog.physicalplan[0][1])
    
    generateDot.generateDot(dlog.physicalplan, "%s.physical.dot"%(name))

    # generate code in the target language
    code = ""
    code += comment("Query " + query)
    code += compile(dlog.physicalplan)
    
    fname = name+'.cpp'
    with open(fname, 'w') as f:
        f.write(code)

    # returns name of code file
    return fname
Exemple #3
0
def main():
    # A simple join
    query = "A(x,z) :- R(x,y), S(y,z)"

    # Triangle
    # query = "A(x,z) :- R(x,y),S(y,z),T(z,x)"

    # Two independent rules
    query = """A(x,z) :- R(x,y), S(y,z); B(x,z) :- S(z,x)."""

    # Create a cmpiler object
    dlog = RACompiler()

    # parse the query
    dlog.fromDatalog(query)

    # Print out the graph
    for (label, root_operator) in dlog.logicalplan:
        graph = root_operator.collectGraph()
        v1 = graph_to_dot(graph)
        v2 = operator_to_dot(root_operator)
        assert v1 == v2
#        print "Dot for individual IDB %s: " % label
#        print v1
#        print

    v3 = plan_to_dot(dlog.logicalplan)
#    print "Dot for combined IDBs:"
    print v3
#    print
    if len(dlog.logicalplan) == 1:
        assert v1 == v3
 def get_phys_plan_root(query, num_server, child_size=None):
     dlog = RACompiler()
     dlog.fromDatalog(query)
     dlog.optimize(myrialang.MyriaHyperCubeAlgebra(
         FakeCatalog(num_server, child_size)))
     ret = dlog.physicalplan
     assert isinstance(ret, algebra.Parallel)
     assert len(ret.children()) == 1
     ret = ret.children()[0]
     assert isinstance(ret, algebra.Store)
     return ret.input
Exemple #5
0
    def execute_query(self,
                      query,
                      test_logical=False,
                      skip_json=False,
                      output="OUTPUT",
                      algebra=MyriaLeftDeepTreeAlgebra):
        """Run a test query against the fake database"""

        dlog = RACompiler()
        dlog.fromDatalog(query)

        assert algebra in [MyriaLeftDeepTreeAlgebra, MyriaHyperCubeAlgebra]

        if test_logical:
            plan = dlog.logicalplan
        else:
            if algebra == MyriaLeftDeepTreeAlgebra:
                dlog.optimize(MyriaLeftDeepTreeAlgebra())
            else:
                dlog.optimize(MyriaHyperCubeAlgebra(FakeCatalog(64)))
            plan = dlog.physicalplan

            if not skip_json:
                # test whether we can generate json without errors
                json_string = json.dumps(
                    compile_to_json(query, dlog.logicalplan, dlog.physicalplan,
                                    "datalog"))
                assert json_string

        self.db.evaluate(plan)
        return self.db.get_table(output)
Exemple #6
0
    def _get_datalog_plan(query, plan_type, algebra, **kwargs):
        datalog = RACompiler()
        datalog.fromDatalog(query)

        if not datalog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")
        elif plan_type == 'logical':
            return datalog.logicalplan
        elif plan_type == 'physical':
            datalog.optimize(target=algebra,
                             push_sql=kwargs.get('push_sql', True))
            return datalog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
def get_plan(query, language, plan_type, connection,
             multiway_join=False, push_sql=False):
    catalog = None
    if multiway_join:
        catalog = MyriaCatalog(connection)
        assert catalog.get_num_servers()
    # Fix up the language string
    if language is None:
        language = "datalog"
    language = language.strip().lower()

    if multiway_join:
        target_algebra = MyriaHyperCubeAlgebra(catalog)
    else:
        target_algebra = MyriaLeftDeepTreeAlgebra()

    if language == "datalog":
        dlog = RACompiler()
        dlog.fromDatalog(query)
        if not dlog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")

        if plan_type == 'logical':
            return dlog.logicalplan
        elif plan_type == 'physical':
            dlog.optimize(target=target_algebra, push_sql=push_sql)
            return dlog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
    elif language in ["myrial", "sql"]:
        # We need a (global) lock on the Myrial parser because yacc
        # .. is not Threadsafe and App Engine uses multiple threads.
        with myrial_parser_lock:
            parsed = myrial_parser.parse(query)
        processor = MyrialInterpreter.StatementProcessor(
            MyriaCatalog(connection))
        processor.evaluate(parsed)
        if plan_type == 'logical':
            return processor.get_physical_plan(target_alg=OptLogicalAlgebra())
        elif plan_type == 'physical':
            return processor.get_physical_plan(target_alg=target_algebra,
                                               multiway_join=multiway_join,
                                               push_sql=push_sql)
        else:
            raise NotImplementedError('Myria plan type %s' % plan_type)

    raise NotImplementedError('Language %s is not supported' % language)
Exemple #8
0
    def execute_query(self, query, test_logical=False, skip_json=False,
                      output="OUTPUT", algebra=MyriaLeftDeepTreeAlgebra):
        """Run a test query against the fake database"""

        dlog = RACompiler()
        dlog.fromDatalog(query)

        assert algebra in [MyriaLeftDeepTreeAlgebra,
                           MyriaHyperCubeAlgebra]

        if test_logical:
            plan = dlog.logicalplan
        else:
            if algebra == MyriaLeftDeepTreeAlgebra:
                dlog.optimize(MyriaLeftDeepTreeAlgebra())
            else:
                dlog.optimize(MyriaHyperCubeAlgebra(FakeCatalog(64)))
            plan = dlog.physicalplan

            if not skip_json:
                # test whether we can generate json without errors
                json_string = json.dumps(compile_to_json(
                    query, dlog.logicalplan, dlog.physicalplan, "datalog"))
                assert json_string

        self.db.evaluate(plan)
        return self.db.get_table(output)
def get_plan(query, language, plan_type, connection):
    # Fix up the language string
    if language is None:
        language = "datalog"
    language = language.strip().lower()

    if language == "datalog":
        dlog = RACompiler()
        dlog.fromDatalog(query)
        if not dlog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")
        if plan_type == 'logical':
            return dlog.logicalplan
        dlog.optimize(target=MyriaAlgebra,
                      eliminate_common_subexpressions=False)
        if plan_type == 'physical':
            return dlog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
    elif language in ["myrial", "sql"]:
        # We need a (global) lock on the Myrial parser because yacc is not
        # ... Threadsafe and App Engine uses multiple threads.
        with myrial_parser_lock:
            parsed = myrial_parser.parse(query)
        processor = MyrialInterpreter.StatementProcessor(
            MyriaCatalog(connection))
        processor.evaluate(parsed)
        if plan_type == 'logical':
            return processor.get_logical_plan()
        elif plan_type == 'physical':
            return processor.get_physical_plan()
        else:
            raise NotImplementedError('Myria plan type %s' % plan_type)
    else:
        raise NotImplementedError('Language %s is not supported' % language)

    raise NotImplementedError('Should not be able to get here')
 def get_phys_plan_root(query, num_server, child_size=None):
     dlog = RACompiler()
     dlog.fromDatalog(query)
     dlog.optimize(
         myrialang.MyriaHyperCubeAlgebra(FakeCatalog(
             num_server, child_size)))
     ret = dlog.physicalplan
     assert isinstance(ret, algebra.Parallel)
     assert len(ret.children()) == 1
     ret = ret.children()[0]
     assert isinstance(ret, algebra.Store)
     return ret.input
Exemple #11
0
def get_plan(query,
             language,
             plan_type,
             connection,
             multiway_join=False,
             push_sql=False):
    catalog = None
    if multiway_join:
        catalog = MyriaCatalog(connection)
        assert catalog.get_num_servers()
    # Fix up the language string
    if language is None:
        language = "datalog"
    language = language.strip().lower()

    if multiway_join:
        target_algebra = MyriaHyperCubeAlgebra(catalog)
    else:
        target_algebra = MyriaLeftDeepTreeAlgebra()

    if language == "datalog":
        dlog = RACompiler()
        dlog.fromDatalog(query)
        if not dlog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")

        if plan_type == 'logical':
            return dlog.logicalplan
        elif plan_type == 'physical':
            dlog.optimize(target=target_algebra, push_sql=push_sql)
            return dlog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
    elif language in ["myrial", "sql"]:
        # We need a (global) lock on the Myrial parser because yacc
        # .. is not Threadsafe and App Engine uses multiple threads.
        with myrial_parser_lock:
            parsed = myrial_parser.parse(query)
        processor = MyrialInterpreter.StatementProcessor(
            MyriaCatalog(connection))
        processor.evaluate(parsed)
        if plan_type == 'logical':
            return processor.get_physical_plan(target_alg=OptLogicalAlgebra())
        elif plan_type == 'physical':
            return processor.get_physical_plan(target_alg=target_algebra,
                                               multiway_join=multiway_join,
                                               push_sql=push_sql)
        else:
            raise NotImplementedError('Myria plan type %s' % plan_type)

    raise NotImplementedError('Language %s is not supported' % language)
Exemple #12
0
    def _get_datalog_plan(query, plan_type, algebra, **kwargs):
        datalog = RACompiler()
        datalog.fromDatalog(query)

        if not datalog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")
        elif plan_type == 'logical':
            return datalog.logicalplan
        elif plan_type == 'physical':
            datalog.optimize(target=algebra,
                             push_sql=kwargs.get('push_sql', True))
            return datalog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
Exemple #13
0
def testEmit(query, name):
    LOG.info("compiling %s: %s", name, query)

    # Create a compiler object
    dlog = RACompiler()

    # parse the query
    dlog.fromDatalog(query)
    #print dlog.parsed
    LOG.info("logical: %s", dlog.logicalplan)

    dlog.optimize(target=GrappaAlgebra)

    LOG.info("physical: %s", dlog.physicalplan[0][1])

    # generate code in the target language
    code = ""
    code += comment("Query " + query)
    code += dlog.compile()

    with open(name + '.cpp', 'w') as f:
        f.write(code)
Exemple #14
0
def emitCode(query, name, algType, plan=None, emit_print=None, dir='.'):
    if emit_print is not None:
        alg = algType(emit_print)
    else:
        alg = algType()

    hack_plan(alg, plan)

    LOG.info("compiling %s: %s", name, query)

    # Create a compiler object
    dlog = RACompiler()

    # parse the query
    dlog.fromDatalog(query)
    # print dlog.parsed
    LOG.info("logical: %s", dlog.logicalplan)

    print dlog.logicalplan
    logical_dot = viz.operator_to_dot(dlog.logicalplan)
    with open("%s.logical.dot" % (name), 'w') as dwf:
        dwf.write(logical_dot)

    dlog.optimize(target=alg)

    LOG.info("physical: %s", dlog.physicalplan)

    print dlog.physicalplan
    physical_dot = viz.operator_to_dot(dlog.physicalplan)
    with open("%s.physical.dot" % (name), 'w') as dwf:
        dwf.write(physical_dot)

    # generate code in the target language
    code = ""
    code += comment("Query " + query)
    code += compile(dlog.physicalplan)

    fname = '{dir}/{name}.cpp'.format(dir=dir, name=name)
    with open(fname, 'w') as f:
        f.write(code)

    # returns name of code file
    return fname
def testEmit(query, name):
    LOG.info("compiling %s: %s", name, query)

    # Create a compiler object
    dlog = RACompiler()

    # parse the query
    dlog.fromDatalog(query)
    #print dlog.parsed
    LOG.info("logical: %s",dlog.logicalplan)

    dlog.optimize(target=GrappaAlgebra)

    LOG.info("physical: %s",dlog.physicalplan[0][1])

    # generate code in the target language
    code = ""
    code += comment("Query " + query)
    code += dlog.compile()

    with open(name+'.cpp', 'w') as f:
        f.write(code)
Exemple #16
0
def get_plan(query, language, plan_type, connection):
    # Fix up the language string
    if language is None:
        language = "datalog"
    language = language.strip().lower()

    if language == "datalog":
        dlog = RACompiler()
        dlog.fromDatalog(query)
        if not dlog.logicalplan:
            raise SyntaxError("Unable to parse Datalog")
        if plan_type == 'logical':
            return dlog.logicalplan
        dlog.optimize(target=MyriaAlgebra,
                      eliminate_common_subexpressions=False)
        if plan_type == 'physical':
            return dlog.physicalplan
        else:
            raise NotImplementedError('Datalog plan type %s' % plan_type)
    elif language in ["myrial", "sql"]:
        # We need a (global) lock on the Myrial parser because yacc is not
        # ... Threadsafe and App Engine uses multiple threads.
        with myrial_parser_lock:
            parsed = myrial_parser.parse(query)
        processor = MyrialInterpreter.StatementProcessor(
            MyriaCatalog(connection))
        processor.evaluate(parsed)
        if plan_type == 'logical':
            return processor.get_logical_plan()
        elif plan_type == 'physical':
            return processor.get_physical_plan()
        else:
            raise NotImplementedError('Myria plan type %s' % plan_type)
    else:
        raise NotImplementedError('Language %s is not supported' % language)

    raise NotImplementedError('Should not be able to get here')
Exemple #17
0
# Compute the cluster centers. First, compute the server-local cluster centers
   CL(cid, avg(x) ) :- C(pid, cid), P(pid, x)

"""

query = """
A@*(x) :- R(x,y,z)
"""

query = """
smallTableJoin(x,z) :- smallTable(x,y),smallTable(y,z)
"""

def comment(s):
  print "/*\n%s\n*/" % str(s)

dlog = RACompiler()

dlog.fromDatalog(query)
print dlog.logicalplan

dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)

code = dlog.compile()
print code

# generate code in the target language
#print compile(physicalplan)
#compile(physicalplan)
print compile(physicalplan)
Exemple #18
0
def RATest(query):
    dlog = RACompiler()
    dlog.fromDatalog(query)
    # TODO: Testing for string equality, but we should do something like what
    # Andrew does -- evaluate the expressions on test data.
    return "%s" % dlog.logicalplan