Ejemplo n.º 1
0
    def getExecutionCode(self):
        assert not self.in_loop, "Bad state, in loop at end of compilation"

        # list -> string
        scan_linearized = emitlist(self.scan_pipelines)

        # Make sure we emitted all the wait statements
        if self.sequence_wait_statements:
            waits_linearized = emitlist(
                list(self.getAndFlushSeqWaitStatements()))
        else:
            waits_linearized = ""

        mem_linearized = \
            emitlist(self.pipelines) + waits_linearized

        flush_linearized = emitlist(self.flush_pipelines)
        scan_linearize_wrap = self.language.group_wrap(gensym(),
                                                       scan_linearized,
                                                       {'type': 'scan'})
        mem_linearize_wrap = self.language.group_wrap(gensym(),
                                                      mem_linearized,
                                                      {'type': 'in_memory'})

        linearized = \
            scan_linearize_wrap + mem_linearize_wrap + flush_linearized

        # substitute all lazily resolved symbols
        resolved = ResolvingSymbol.substitute(linearized,
                                              self.resolving_symbols)

        return resolved
Ejemplo n.º 2
0
    def getExecutionCode(self):
        assert not self.in_loop, "Bad state, in loop at end of compilation"

        # list -> string
        scan_linearized = emitlist(self.scan_pipelines)

        # Make sure we emitted all the wait statements
        if self.sequence_wait_statements:
            waits_linearized = emitlist(
                list(self.getAndFlushSeqWaitStatements()))
        else:
            waits_linearized = ""

        mem_linearized = \
            emitlist(self.pipelines) + waits_linearized

        flush_linearized = emitlist(self.flush_pipelines)
        scan_linearize_wrap = self.language.group_wrap(gensym(),
                                                       scan_linearized,
                                                       {'type': 'scan'})
        mem_linearize_wrap = self.language.group_wrap(gensym(), mem_linearized,
                                                      {'type': 'in_memory'})

        linearized = \
            scan_linearize_wrap + mem_linearize_wrap + flush_linearized

        # substitute all lazily resolved symbols
        resolved = ResolvingSymbol.substitute(linearized,
                                              self.resolving_symbols)

        return resolved
Ejemplo n.º 3
0
  def consume(self, t, src, state):
    if src.childtag == "right":
      declr_template =  """std::unordered_map<int64_t, std::vector<%(in_tuple_type)s>* > %(hashname)s;
      """
      
      right_template = """insert(%(hashname)s, %(keyname)s, %(keypos)s);
      """   

      hashname = self._hashname
      keyname = t.name

      # find the attribute that corresponds to the right child
      if self.rightCondIsRightAttr:
        keypos = self.condition.right.position-len(self.left.scheme())
      else:
        keypos = self.condition.left.position-len(self.left.scheme())


      in_tuple_type = t.getTupleTypename()

      # declaration of hash map
      hashdeclr =  declr_template % locals()
      state.addDeclarations([hashdeclr])
      
      # materialization point
      code = right_template % locals()
      
      return code
    
    if src.childtag == "left":
      left_template = """
      for (auto %(right_tuple_name)s : lookup(%(hashname)s, %(keyname)s.get(%(keypos)s))) {
        auto %(out_tuple_name)s = combine<%(out_tuple_type)s> (%(keyname)s, %(right_tuple_name)s);
     %(inner_plan_compiled)s 
  }
  """
      hashname = self._hashname
      keyname = t.name
      keytype = t.getTupleTypename()

      if self.rightCondIsRightAttr:
          keypos = self.condition.left.position
      else:
          keypos = self.condition.right.position

      right_tuple_name = gensym()

      outTuple = CStagedTupleRef(gensym(), self.scheme())
      out_tuple_type_def = outTuple.generateDefinition()
      out_tuple_type = outTuple.getTupleTypename()
      out_tuple_name = outTuple.name

      state.addDeclarations([out_tuple_type_def])

      inner_plan_compiled = self.parent.consume(outTuple, self, state)
      
      code = left_template % locals()
      return code

    assert False, "src not equal to left or right"
Ejemplo n.º 4
0
    def getExecutionCode(self):
        # list -> string
        scan_linearized = emitlist(self.scan_pipelines)
        mem_linearized = emitlist(self.pipelines)

        scan_linearized_wrapped = self.language.group_wrap(gensym(), scan_linearized, {'type': 'scan'})
        mem_linearized_wrapped = self.language.group_wrap(gensym(), mem_linearized, {'type': 'in_memory'})

        linearized = scan_linearized_wrapped + mem_linearized_wrapped

        # substitute all lazily resolved symbols
        resolved = linearized % self.resolving_symbols

        return resolved
Ejemplo n.º 5
0
    def produce(self, state):

        # Common subexpression elimination
        # don't scan the same file twice
        resultsym = state.lookupExpr(self)
        LOG.debug("lookup %s(h=%s) => %s", self, self.__hash__(), resultsym)
        if not resultsym:
            #TODO for now this will break whatever relies on self.bound like reusescans
            #Scan is the only place where a relation is declared
            resultsym = gensym()

            fscode = self.__compileme__(resultsym)

            state.saveExpr(self, resultsym)

            stagedTuple = self.new_tuple_ref(resultsym, self.scheme())
            state.saveTupleDef(resultsym, stagedTuple)

            tuple_type_def = stagedTuple.generateDefinition()
            tuple_type = stagedTuple.getTupleTypename()
            state.addDeclarations([tuple_type_def])

            rel_decl_template = self.__get_relation_decl_template__()
            if rel_decl_template:
                state.addDeclarations([rel_decl_template % locals()])

            # now that we have the type, format this in;
            state.setPipelineProperty('type', 'scan')
            state.addPipeline(fscode%{"result_type": tuple_type})


        # no return value used because parent is a new pipeline
        self.parent.consume(resultsym, self, state)
Ejemplo n.º 6
0
  def produce(self, state):
    # declare a single new type for project
    #TODO: instead do mark used-columns?

    # always does an assignment to new tuple
    self.newtuple = self.new_tuple_ref(gensym(), self.scheme())
    state.addDeclarations( [self.newtuple.generateDefinition()] )

    self.input.produce(state)
Ejemplo n.º 7
0
    def getExecutionCode(self):
        # list -> string
        scan_linearized = emitlist(self.scan_pipelines)
        mem_linearized = \
            emitlist(self.pipelines) + emitlist(self.main_wait_statements)
        flush_linearized = emitlist(self.flush_pipelines)
        scan_linearize_wrap = self.language.group_wrap(gensym(),
                                                       scan_linearized,
                                                       {'type': 'scan'})
        mem_linearize_wrap = self.language.group_wrap(gensym(),
                                                      mem_linearized,
                                                      {'type': 'in_memory'})

        linearized = \
            scan_linearize_wrap + mem_linearize_wrap + flush_linearized

        # substitute all lazily resolved symbols
        resolved = linearized % self.resolving_symbols

        return resolved
Ejemplo n.º 8
0
    def produce(self, state):
        self.syncnames = []

        if not isinstance(self.condition, expression.EQ):
            msg = "The C compiler can only handle equi-join conditions\
             of a single attribute: %s" % self.condition
            raise ValueError(msg)

        init_template = ct("""%(hashname)s.init_global_DHT( &%(hashname)s, \
        cores()*16*1024 );
                        """)
        declr_template = ct("""typedef DoubleDHT<int64_t, \
                                                   %(left_in_tuple_type)s, \
                                                   %(right_in_tuple_type)s,
                                                std_hash> \
                    DHT_%(left_in_tuple_type)s_%(right_in_tuple_type)s;
      DHT_%(left_in_tuple_type)s_%(right_in_tuple_type)s %(hashname)s;
      """)
        # declaration of hash map
        self._hashname = self.__genHashName__()
        hashname = self._hashname
        self.leftTypeRef = state.createUnresolvedSymbol()
        left_in_tuple_type = self.leftTypeRef.getPlaceholder()
        self.rightTypeRef = state.createUnresolvedSymbol()
        right_in_tuple_type = self.rightTypeRef.getPlaceholder()
        hashdeclr = declr_template % locals()

        state.addDeclarationsUnresolved([hashdeclr])

        self.outTuple = GrappaStagedTupleRef(gensym(), self.scheme())
        out_tuple_type_def = self.outTuple.generateDefinition()
        state.addDeclarations([out_tuple_type_def])

        # find the attribute that corresponds to the right child
        self.rightCondIsRightAttr = \
            self.condition.right.position >= len(self.left.scheme())
        self.leftCondIsRightAttr = \
            self.condition.left.position >= len(self.left.scheme())
        assert self.rightCondIsRightAttr ^ self.leftCondIsRightAttr

        self.right.childtag = "right"
        state.addInitializers([init_template % locals()])
        self.right.produce(state)

        self.left.childtag = "left"
        self.left.produce(state)

        for sn in self.syncnames:
            syncname = sn
            state.addCode(self.wait_template % locals())
Ejemplo n.º 9
0
    def consume(self, inputsym, src, state):
        # generate the materialization from file into memory

        # scan from index
        # memory_scan_template = """forall_localized( %(inputsym)s_index->vs, \
        # %(inputsym)s_index->nv, [](int64_t ai, Vertex& a) {
        #      forall_here_async<&impl::local_gce>( 0, a.nadj, \
        # [=](int64_t start, int64_t iters) {
        #      for (int64_t i=start; i<start+iters; i++) {
        #        auto %(tuple_name)s = a.local_adj[i];
        #
        #          %(inner_plan_compiled)s
        #       } // end scan over %(inputsym)s (for)
        #       }); // end scan over %(inputsym)s (forall_here_async)
        #       }); // end scan over %(inputsym)s (forall_localized)
        #       """

        global_sync_decl_template = ct("""
        GlobalCompletionEvent %(global_syncname)s;
        """)

        global_syncname = gensym()
        state.addDeclarations([global_sync_decl_template % locals()])
        state.setPipelineProperty('global_syncname', global_syncname)

        memory_scan_template = ct("""
    forall<&%(global_syncname)s>( %(inputsym)s.data, %(inputsym)s.numtuples, \
    [=](int64_t i, %(tuple_type)s& %(tuple_name)s) {
    %(inner_plan_compiled)s
    }); // end  scan over %(inputsym)s
    """)

        stagedTuple = state.lookupTupleDef(inputsym)
        tuple_type = stagedTuple.getTupleTypename()
        tuple_name = stagedTuple.name

        inner_plan_compiled = self.parent.consume(stagedTuple, self, state)

        code = memory_scan_template % locals()
        state.setPipelineProperty('type', 'in_memory')
        state.addPipeline(code)
        return None
Ejemplo n.º 10
0
 def createUnresolvedSymbol(self):
     name = gensym()
     rs = ResolvingSymbol(name)
     self.resolving_symbols[name] = None
     return rs
Ejemplo n.º 11
0
    def consume(self, t, src, state):
        if src.childtag == "right":

            right_template = ct("""
            %(hashname)s.insert(%(keyname)s.get(%(keypos)s), %(keyname)s);
            """)

            hashname = self._hashname
            keyname = t.name

            if self.rightCondIsRightAttr:
                keypos = self.condition.right.position \
                    - len(self.left.scheme())
            else:
                keypos = self.condition.left.position \
                    - len(self.left.scheme())

            self.rightTupleTypename = t.getTupleTypename()
            if self.rightTupleTypeRef is not None:
                state.resolveSymbol(self.rightTupleTypeRef,
                                    self.rightTupleTypename)

            # materialization point
            code = right_template % locals()

            return code

        if src.childtag == "left":
            left_template = ct("""
            %(hashname)s.lookup_iter<&%(pipeline_sync)s>( \
            %(keyname)s.get(%(keypos)s), \
            [=](%(right_tuple_type)s& %(right_tuple_name)s) {
              join_coarse_result_count++;
              %(out_tuple_type)s %(out_tuple_name)s = \
               combine<%(out_tuple_type)s, \
                       %(keytype)s, \
                       %(right_tuple_type)s> \
                           (%(keyname)s, %(right_tuple_name)s);
              %(inner_plan_compiled)s
            });
     """)

            hashname = self._hashname
            keyname = t.name
            keytype = t.getTupleTypename()

            pipeline_sync = state.getPipelineProperty('global_syncname')

            if self.rightCondIsRightAttr:
                keypos = self.condition.left.position
            else:
                keypos = self.condition.right.position

            right_tuple_name = gensym()
            right_tuple_type = self.rightTupleTypename

            outTuple = GrappaStagedTupleRef(gensym(), self.scheme())
            out_tuple_type_def = outTuple.generateDefinition()
            out_tuple_type = outTuple.getTupleTypename()
            out_tuple_name = outTuple.name

            state.addDeclarations([out_tuple_type_def])

            inner_plan_compiled = self.parent.consume(outTuple, self, state)

            code = left_template % locals()
            return code

        assert False, "src not equal to left or right"
Ejemplo n.º 12
0
    def consume(self, t, src, state):
        access_template = ct("""
        %(hashname)s.insert_lookup_iter_%(side)s<&%(global_syncname)s>(\
        %(keyname)s.get(%(keypos)s), %(keyname)s, \
        [=](%(other_tuple_type)s %(valname)s) {
            join_coarse_result_count++;
            %(out_tuple_type)s %(out_tuple_name)s = \
                             combine<%(out_tuple_type)s, \
                                      %(left_type)s, \
                                      %(right_type)s> (%(left_name)s, \
                            %(right_name)s);
                                %(inner_plan_compiled)s
                                });
                                """)

        hashname = self._hashname
        keyname = t.name
        side = src.childtag

        outTuple = self.outTuple
        out_tuple_type = self.outTuple.getTupleTypename()
        out_tuple_name = self.outTuple.name

        syncname = self.__genSyncName__()
        state.setPipelineProperty('sync', syncname)
        state.setPipelineProperty('syncdef', syncname)
        self.syncnames.append(syncname)

        global_syncname = state.getPipelineProperty('global_syncname')

        if src.childtag == "right":

            # save for later
            self.right_in_tuple_type = t.getTupleTypename()
            state.resolveSymbol(self.rightTypeRef, self.right_in_tuple_type)

            if self.rightCondIsRightAttr:
                keypos = self.condition.right.position \
                    - len(self.left.scheme())
            else:
                keypos = self.condition.left.position \
                    - len(self.left.scheme())

            inner_plan_compiled = self.parent.consume(outTuple, self, state)

            other_tuple_type = self.leftTypeRef.getPlaceholder()
            left_type = other_tuple_type
            right_type = self.right_in_tuple_type
            left_name = gensym()
            right_name = keyname
            self.right_name = right_name
            valname = left_name

            code = access_template % locals()
            return code

        if src.childtag == "left":
            right_in_tuple_type = self.right_in_tuple_type
            left_in_tuple_type = t.getTupleTypename()
            state.resolveSymbol(self.leftTypeRef, left_in_tuple_type)

            if self.rightCondIsRightAttr:
                keypos = self.condition.left.position
            else:
                keypos = self.condition.right.position

            inner_plan_compiled = self.parent.consume(outTuple, self, state)

            left_type = left_in_tuple_type
            right_type = self.right_in_tuple_type
            other_tuple_type = self.right_in_tuple_type
            left_name = keyname
            right_name = gensym()
            valname = right_name

            code = access_template % locals()
            return code

        assert False, "src not equal to left or right"
Ejemplo n.º 13
0
  def produce(self, state):
    self.unifiedTupleType = self.new_tuple_ref(gensym(), self.scheme())
    state.addDeclarations([self.unifiedTupleType.generateDefinition()])

    self.right.produce(state)
    self.left.produce(state)
Ejemplo n.º 14
0
 def createUnresolvedSymbol(self):
     name = gensym()
     rs = ResolvingSymbol(name)
     self.resolving_symbols[name] = None
     return rs