def get_refined_updated_query(self, qid, ref_level, prev_qid=0, prev_ref_level=0, has_join=False, join_qid=None):
# return query with updated threshold values and map operation---masking based on refinement level
if self.is_refinement_enabled:
iter_qids = self.refined_sonata_queries[qid][ref_level].keys()
iter_qids.sort()
tmp_query = self.refined_sonata_queries[qid][ref_level][iter_qids[-1]]
if prev_ref_level > 0:
out_query = PacketStream(tmp_query.qid)
out_query.basic_headers = get_concise_headers(tmp_query)
# print (not has_join), prev_qid, prev_ref_level, qid, ref_level
if not has_join:
refined_qid_src = 10000 * prev_qid + prev_ref_level
out_query.filter(append_type=1, src=refined_qid_src, filter_keys=(self.per_query_refinement_key[qid], ),
func=('mask', prev_ref_level,))
else:
if prev_qid == qid:
refined_qid_src = 10000 * join_qid + prev_ref_level
out_query.filter(append_type=1, src=refined_qid_src, filter_keys=(self.per_query_refinement_key[qid], ),
func=('mask', prev_ref_level,))
for operator in tmp_query.operators:
copy_operators(out_query, operator)
else:
out_query = tmp_query
else:
out_query = PacketStream(self.query.qid)
out_query.basic_headers = get_concise_headers(self.query)
for operator in self.query.operators:
copy_operators(out_query, operator)
# out_query = self.query
return out_query
def generate_partitioned_queries_learning(self):
sonata_query = self.query
partition_plans_learning = self.get_partition_plans_learning(
sonata_query)
# print partition_plans_learning
intermediate_learning_queries = {}
prev_qid = 0
filter_mappings = {}
filters_marked = {}
for max_operators in partition_plans_learning:
qid = 1000 * sonata_query.qid + max_operators
tmp_query = (PacketStream(sonata_query.qid))
# tmp_query.basic_headers = BASIC_HEADERS
ctr = 0
filter_ctr = 0
prev_operator = None
for operator in sonata_query.operators:
can_increment = True
if operator.name != 'Join':
if ctr < max_operators:
copy_operators(tmp_query, operator)
prev_operator = operator
else:
break
if operator.name == 'Filter':
filter_ctr += 1
if (qid, self.ref_level,
filter_ctr) not in filters_marked:
filters_marked[(
qid,
self.ref_level,
filter_ctr,
)] = sonata_query.qid
filter_mappings[(prev_qid, qid,
self.ref_level)] = (
sonata_query.qid, filter_ctr,
operator.func[1])
else:
prev_operator = operator
copy_operators(tmp_query, operator)
if operator.name == 'Map':
if hasattr(operator, 'func') and len(operator.func) > 0:
if operator.func[0] == 'mask':
can_increment = False
if can_increment:
ctr += 1
intermediate_learning_queries[qid] = tmp_query
prev_qid = qid
self.intermediate_learning_queries = intermediate_learning_queries
self.filter_mappings = filter_mappings
def apply_refinement_plan(sonata_query, refinement_key, refined_query_id, ref_level):
# base refined query + headers
refined_sonata_query = PacketStream(refined_query_id)
refined_sonata_query.basic_headers = get_concise_headers(sonata_query)
# Add refinement level, eg: 32, 24
refined_sonata_query.map(map_keys=(refinement_key,), func=("mask", ref_level))
# Copy operators to the new refined sonata query
for operator in sonata_query.operators:
# print "apply_refinement_plan: ", operator.name
copy_operators(refined_sonata_query, operator)
return refined_sonata_query