def getExecutionCode(self):
assert not self.in_loop, "Bad state, in loop at end of compilation"
# list -> string
scan_linearized = emitlist(self.scan_pipelines)
# Make sure we emitted all the wait statements
if self.sequence_wait_statements:
waits_linearized = emitlist(
list(self.getAndFlushSeqWaitStatements()))
else:
waits_linearized = ""
mem_linearized = \
emitlist(self.pipelines) + waits_linearized
flush_linearized = emitlist(self.flush_pipelines)
scan_linearize_wrap = self.language.group_wrap(gensym(),
scan_linearized,
{'type': 'scan'})
mem_linearize_wrap = self.language.group_wrap(gensym(),
mem_linearized,
{'type': 'in_memory'})
linearized = \
scan_linearize_wrap + mem_linearize_wrap + flush_linearized
# substitute all lazily resolved symbols
resolved = ResolvingSymbol.substitute(linearized,
self.resolving_symbols)
return resolved
def getExecutionCode(self):
assert not self.in_loop, "Bad state, in loop at end of compilation"
# list -> string
scan_linearized = emitlist(self.scan_pipelines)
# Make sure we emitted all the wait statements
if self.sequence_wait_statements:
waits_linearized = emitlist(
list(self.getAndFlushSeqWaitStatements()))
else:
waits_linearized = ""
mem_linearized = \
emitlist(self.pipelines) + waits_linearized
flush_linearized = emitlist(self.flush_pipelines)
scan_linearize_wrap = self.language.group_wrap(gensym(),
scan_linearized,
{'type': 'scan'})
mem_linearize_wrap = self.language.group_wrap(gensym(), mem_linearized,
{'type': 'in_memory'})
linearized = \
scan_linearize_wrap + mem_linearize_wrap + flush_linearized
# substitute all lazily resolved symbols
resolved = ResolvingSymbol.substitute(linearized,
self.resolving_symbols)
return resolved
def consume(self, t, src, state):
if src.childtag == "right":
declr_template = """std::unordered_map<int64_t, std::vector<%(in_tuple_type)s>* > %(hashname)s;
"""
right_template = """insert(%(hashname)s, %(keyname)s, %(keypos)s);
"""
hashname = self._hashname
keyname = t.name
# find the attribute that corresponds to the right child
if self.rightCondIsRightAttr:
keypos = self.condition.right.position-len(self.left.scheme())
else:
keypos = self.condition.left.position-len(self.left.scheme())
in_tuple_type = t.getTupleTypename()
# declaration of hash map
hashdeclr = declr_template % locals()
state.addDeclarations([hashdeclr])
# materialization point
code = right_template % locals()
return code
if src.childtag == "left":
left_template = """
for (auto %(right_tuple_name)s : lookup(%(hashname)s, %(keyname)s.get(%(keypos)s))) {
auto %(out_tuple_name)s = combine<%(out_tuple_type)s> (%(keyname)s, %(right_tuple_name)s);
%(inner_plan_compiled)s
}
"""
hashname = self._hashname
keyname = t.name
keytype = t.getTupleTypename()
if self.rightCondIsRightAttr:
keypos = self.condition.left.position
else:
keypos = self.condition.right.position
right_tuple_name = gensym()
outTuple = CStagedTupleRef(gensym(), self.scheme())
out_tuple_type_def = outTuple.generateDefinition()
out_tuple_type = outTuple.getTupleTypename()
out_tuple_name = outTuple.name
state.addDeclarations([out_tuple_type_def])
inner_plan_compiled = self.parent.consume(outTuple, self, state)
code = left_template % locals()
return code
assert False, "src not equal to left or right"
def getExecutionCode(self):
# list -> string
scan_linearized = emitlist(self.scan_pipelines)
mem_linearized = emitlist(self.pipelines)
scan_linearized_wrapped = self.language.group_wrap(gensym(), scan_linearized, {'type': 'scan'})
mem_linearized_wrapped = self.language.group_wrap(gensym(), mem_linearized, {'type': 'in_memory'})
linearized = scan_linearized_wrapped + mem_linearized_wrapped
# substitute all lazily resolved symbols
resolved = linearized % self.resolving_symbols
return resolved
def produce(self, state):
# Common subexpression elimination
# don't scan the same file twice
resultsym = state.lookupExpr(self)
LOG.debug("lookup %s(h=%s) => %s", self, self.__hash__(), resultsym)
if not resultsym:
#TODO for now this will break whatever relies on self.bound like reusescans
#Scan is the only place where a relation is declared
resultsym = gensym()
fscode = self.__compileme__(resultsym)
state.saveExpr(self, resultsym)
stagedTuple = self.new_tuple_ref(resultsym, self.scheme())
state.saveTupleDef(resultsym, stagedTuple)
tuple_type_def = stagedTuple.generateDefinition()
tuple_type = stagedTuple.getTupleTypename()
state.addDeclarations([tuple_type_def])
rel_decl_template = self.__get_relation_decl_template__()
if rel_decl_template:
state.addDeclarations([rel_decl_template % locals()])
# now that we have the type, format this in;
state.setPipelineProperty('type', 'scan')
state.addPipeline(fscode%{"result_type": tuple_type})
# no return value used because parent is a new pipeline
self.parent.consume(resultsym, self, state)
def produce(self, state):
# declare a single new type for project
#TODO: instead do mark used-columns?
# always does an assignment to new tuple
self.newtuple = self.new_tuple_ref(gensym(), self.scheme())
state.addDeclarations( [self.newtuple.generateDefinition()] )
self.input.produce(state)
def getExecutionCode(self):
# list -> string
scan_linearized = emitlist(self.scan_pipelines)
mem_linearized = \
emitlist(self.pipelines) + emitlist(self.main_wait_statements)
flush_linearized = emitlist(self.flush_pipelines)
scan_linearize_wrap = self.language.group_wrap(gensym(),
scan_linearized,
{'type': 'scan'})
mem_linearize_wrap = self.language.group_wrap(gensym(),
mem_linearized,
{'type': 'in_memory'})
linearized = \
scan_linearize_wrap + mem_linearize_wrap + flush_linearized
# substitute all lazily resolved symbols
resolved = linearized % self.resolving_symbols
return resolved
def produce(self, state):
self.syncnames = []
if not isinstance(self.condition, expression.EQ):
msg = "The C compiler can only handle equi-join conditions\
of a single attribute: %s" % self.condition
raise ValueError(msg)
init_template = ct("""%(hashname)s.init_global_DHT( &%(hashname)s, \
cores()*16*1024 );
""")
declr_template = ct("""typedef DoubleDHT<int64_t, \
%(left_in_tuple_type)s, \
%(right_in_tuple_type)s,
std_hash> \
DHT_%(left_in_tuple_type)s_%(right_in_tuple_type)s;
DHT_%(left_in_tuple_type)s_%(right_in_tuple_type)s %(hashname)s;
""")
# declaration of hash map
self._hashname = self.__genHashName__()
hashname = self._hashname
self.leftTypeRef = state.createUnresolvedSymbol()
left_in_tuple_type = self.leftTypeRef.getPlaceholder()
self.rightTypeRef = state.createUnresolvedSymbol()
right_in_tuple_type = self.rightTypeRef.getPlaceholder()
hashdeclr = declr_template % locals()
state.addDeclarationsUnresolved([hashdeclr])
self.outTuple = GrappaStagedTupleRef(gensym(), self.scheme())
out_tuple_type_def = self.outTuple.generateDefinition()
state.addDeclarations([out_tuple_type_def])
# find the attribute that corresponds to the right child
self.rightCondIsRightAttr = \
self.condition.right.position >= len(self.left.scheme())
self.leftCondIsRightAttr = \
self.condition.left.position >= len(self.left.scheme())
assert self.rightCondIsRightAttr ^ self.leftCondIsRightAttr
self.right.childtag = "right"
state.addInitializers([init_template % locals()])
self.right.produce(state)
self.left.childtag = "left"
self.left.produce(state)
for sn in self.syncnames:
syncname = sn
state.addCode(self.wait_template % locals())
def consume(self, inputsym, src, state):
# generate the materialization from file into memory
# scan from index
# memory_scan_template = """forall_localized( %(inputsym)s_index->vs, \
# %(inputsym)s_index->nv, [](int64_t ai, Vertex& a) {
# forall_here_async<&impl::local_gce>( 0, a.nadj, \
# [=](int64_t start, int64_t iters) {
# for (int64_t i=start; i<start+iters; i++) {
# auto %(tuple_name)s = a.local_adj[i];
#
# %(inner_plan_compiled)s
# } // end scan over %(inputsym)s (for)
# }); // end scan over %(inputsym)s (forall_here_async)
# }); // end scan over %(inputsym)s (forall_localized)
# """
global_sync_decl_template = ct("""
GlobalCompletionEvent %(global_syncname)s;
""")
global_syncname = gensym()
state.addDeclarations([global_sync_decl_template % locals()])
state.setPipelineProperty('global_syncname', global_syncname)
memory_scan_template = ct("""
forall<&%(global_syncname)s>( %(inputsym)s.data, %(inputsym)s.numtuples, \
[=](int64_t i, %(tuple_type)s& %(tuple_name)s) {
%(inner_plan_compiled)s
}); // end scan over %(inputsym)s
""")
stagedTuple = state.lookupTupleDef(inputsym)
tuple_type = stagedTuple.getTupleTypename()
tuple_name = stagedTuple.name
inner_plan_compiled = self.parent.consume(stagedTuple, self, state)
code = memory_scan_template % locals()
state.setPipelineProperty('type', 'in_memory')
state.addPipeline(code)
return None
def createUnresolvedSymbol(self):
name = gensym()
rs = ResolvingSymbol(name)
self.resolving_symbols[name] = None
return rs
def consume(self, t, src, state):
if src.childtag == "right":
right_template = ct("""
%(hashname)s.insert(%(keyname)s.get(%(keypos)s), %(keyname)s);
""")
hashname = self._hashname
keyname = t.name
if self.rightCondIsRightAttr:
keypos = self.condition.right.position \
- len(self.left.scheme())
else:
keypos = self.condition.left.position \
- len(self.left.scheme())
self.rightTupleTypename = t.getTupleTypename()
if self.rightTupleTypeRef is not None:
state.resolveSymbol(self.rightTupleTypeRef,
self.rightTupleTypename)
# materialization point
code = right_template % locals()
return code
if src.childtag == "left":
left_template = ct("""
%(hashname)s.lookup_iter<&%(pipeline_sync)s>( \
%(keyname)s.get(%(keypos)s), \
[=](%(right_tuple_type)s& %(right_tuple_name)s) {
join_coarse_result_count++;
%(out_tuple_type)s %(out_tuple_name)s = \
combine<%(out_tuple_type)s, \
%(keytype)s, \
%(right_tuple_type)s> \
(%(keyname)s, %(right_tuple_name)s);
%(inner_plan_compiled)s
});
""")
hashname = self._hashname
keyname = t.name
keytype = t.getTupleTypename()
pipeline_sync = state.getPipelineProperty('global_syncname')
if self.rightCondIsRightAttr:
keypos = self.condition.left.position
else:
keypos = self.condition.right.position
right_tuple_name = gensym()
right_tuple_type = self.rightTupleTypename
outTuple = GrappaStagedTupleRef(gensym(), self.scheme())
out_tuple_type_def = outTuple.generateDefinition()
out_tuple_type = outTuple.getTupleTypename()
out_tuple_name = outTuple.name
state.addDeclarations([out_tuple_type_def])
inner_plan_compiled = self.parent.consume(outTuple, self, state)
code = left_template % locals()
return code
assert False, "src not equal to left or right"
def consume(self, t, src, state):
access_template = ct("""
%(hashname)s.insert_lookup_iter_%(side)s<&%(global_syncname)s>(\
%(keyname)s.get(%(keypos)s), %(keyname)s, \
[=](%(other_tuple_type)s %(valname)s) {
join_coarse_result_count++;
%(out_tuple_type)s %(out_tuple_name)s = \
combine<%(out_tuple_type)s, \
%(left_type)s, \
%(right_type)s> (%(left_name)s, \
%(right_name)s);
%(inner_plan_compiled)s
});
""")
hashname = self._hashname
keyname = t.name
side = src.childtag
outTuple = self.outTuple
out_tuple_type = self.outTuple.getTupleTypename()
out_tuple_name = self.outTuple.name
syncname = self.__genSyncName__()
state.setPipelineProperty('sync', syncname)
state.setPipelineProperty('syncdef', syncname)
self.syncnames.append(syncname)
global_syncname = state.getPipelineProperty('global_syncname')
if src.childtag == "right":
# save for later
self.right_in_tuple_type = t.getTupleTypename()
state.resolveSymbol(self.rightTypeRef, self.right_in_tuple_type)
if self.rightCondIsRightAttr:
keypos = self.condition.right.position \
- len(self.left.scheme())
else:
keypos = self.condition.left.position \
- len(self.left.scheme())
inner_plan_compiled = self.parent.consume(outTuple, self, state)
other_tuple_type = self.leftTypeRef.getPlaceholder()
left_type = other_tuple_type
right_type = self.right_in_tuple_type
left_name = gensym()
right_name = keyname
self.right_name = right_name
valname = left_name
code = access_template % locals()
return code
if src.childtag == "left":
right_in_tuple_type = self.right_in_tuple_type
left_in_tuple_type = t.getTupleTypename()
state.resolveSymbol(self.leftTypeRef, left_in_tuple_type)
if self.rightCondIsRightAttr:
keypos = self.condition.left.position
else:
keypos = self.condition.right.position
inner_plan_compiled = self.parent.consume(outTuple, self, state)
left_type = left_in_tuple_type
right_type = self.right_in_tuple_type
other_tuple_type = self.right_in_tuple_type
left_name = keyname
right_name = gensym()
valname = right_name
code = access_template % locals()
return code
assert False, "src not equal to left or right"
def produce(self, state):
self.unifiedTupleType = self.new_tuple_ref(gensym(), self.scheme())
state.addDeclarations([self.unifiedTupleType.generateDefinition()])
self.right.produce(state)
self.left.produce(state)
def createUnresolvedSymbol(self):
name = gensym()
rs = ResolvingSymbol(name)
self.resolving_symbols[name] = None
return rs
