def deploy(self):
queue = [self]
# Once the DAG is set, mark the final operator as the final. This helps
# any type conversions from tuples to Table (right now, only used for
# the MultiOperator).
num_ends = 0
seen = set()
while len(queue) > 0:
op = queue.pop(0)
if len(op.downstreams) == 0 and op.fn_name not in seen:
num_ends += 1
op.final = True
seen.add(op.fn_name)
queue.extend(op.downstreams)
if num_ends > 1:
raise FlowError('You must converge all of your operators to a' +
' single output.')
functions, connections, gpus, batching = [], [], [], []
for downstream in self.downstreams:
fns, conns, args, _, registered, ds_gpus, ds_batching = \
downstream.deploy(self.cloudburst)
functions += fns
connections += conns
gpus += ds_gpus
batching += ds_batching
self.registered = registered
if len(functions) > 0:
uid = str(len(registered))
name = self.flowname + '-' + uid
registered[name] = args
success, error = self.cloudburst.register_dag(
name,
functions,
connections,
colocated=self.colocates,
gpu_functions=gpus,
batching_functions=batching)
if not success:
raise FlowError(str(error))
self.deployed = True
def __setitem__(self, key, value):
if key == Row.qid_key:
raise FlowError('Cannot modify query ID of a row.')
for name, tp in self.schema:
if key == name:
if not isinstance(value, tp.typ):
raise FlowError(
f'Invalid update to {key}: Does not match' +
f'type {tp}.')
else:
break
self.data[key] = value
def _check(self, vals: List[Any]):
if len(vals) != len(self.schema):
raise FlowError(f'Expected {len(self.schema)} vals but found' +
f' {len(vals)}')
for idx, val in enumerate(vals):
name, typ = self.schema[idx]
# None values are okay because they are NULLs.
if val is not None and not isinstance(val, typ.typ):
raise FlowError(f'Expected type {typ.typ} but instead found' +
f' {type(val)} for column {name}')
return True
def average(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
' field.')
sm = 0.0
for row in table.get():
sm += row[column]
return sm / table.size()
def insert(self, vals: List[Any], qid: int = None) -> bool:
if isinstance(vals, Row):
if vals.schema == self.schema:
self.data.append(vals)
return True
raise FlowError(f'Invalid row insertion: {vals}')
if not isinstance(vals, list):
raise FlowError('Unrecognized type: ' + str(vals) +
'\n\nCan only insert a Row or a list.')
if not self._check(vals):
return False
if qid is None:
self.data.append(Row(self.schema, vals, self.qid_count))
self.qid_count += 1
else:
self.data.append(Row(self.schema, vals, qid))
return True
def max(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
'field.')
mx = None
for row in table.get():
if mx is None:
mx = row[column]
if row[column] > mx:
mx = row[column]
return mx
def merge_tables(tables: List[Table]) -> (Table, Dict[int, int]):
schema = tables[0].schema
for other in tables[1:]:
if other.schema != schema:
raise FlowError(f'Schema mismatch:\n\t{schema}\n\t{other.schema}')
mappings = {}
mappings['NUM_TABLES'] = len(tables)
result = Table(schema)
qid = 0
for idx, table in enumerate(tables):
for row in table.get():
# Convert to a list, so it gets assigned a new qid.
vals = []
for val, _ in schema:
vals.append(row[val])
result.insert(vals, qid)
mappings[qid] = idx
qid += 1
return result, mappings
def __init__(self, flowname: str, aggregate: str, column: str, sink: list):
if aggregate not in AGGREGATES:
raise FlowError(f'Unknown aggregate: {aggregate}')
if aggregate != 'count' and column is None:
raise FlowError(f'For non-count aggregates, column must be' +
' specified.')
self.__name__ = 'AggregateOperator'
def x(x: int):
x + 1 # XXX: This is a hack for registration.
self.fn = x
self._setup(flowname)
self.aggregate = aggregate
self.column = column
self.sink = sink
class AggregateLogic:
def __init__(self, cloudburst):
self.preprocess(cloudburst)
def preprocess(self, _):
pass
def run(self, cloudburst, aggregate, column, inp):
serialized = False
if type(inp) == bytes:
serialized = True
inp = deserialize(inp)
if aggregate == 'count':
aggfn = self.count
if aggregate == 'min':
aggfn = self.min
if aggregate == 'max':
aggfn = self.max
if aggregate == 'sum':
aggfn = self.sum
if aggregate == 'average':
aggfn = self.average
if isinstance(inp, GroupbyTable):
gb_col = inp.col
val, _ = next(inp.get())
gb_typ = get_type(type(val))
result = Table([(gb_col, gb_typ), (aggregate, FloatType)])
for val, tbl in inp.get():
agg = aggfn(tbl, column)
result.insert([val, float(agg)])
else:
result = Table([(aggregate, FloatType)])
result.insert([float(aggnf(inp, column))])
if serialized:
result = serialize(result)
return result
def count(self, table, column):
return table.size()
def min(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
' field.')
mn = None
for row in table.get():
if mn is None:
mn = row[column]
if row[column] < mn:
mn = row[column]
return mn
def max(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
'field.')
mx = None
for row in table.get():
if mx is None:
mx = row[column]
if row[column] > mx:
mx = row[column]
return mx
def sum(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
' field.')
sm = 0.0
for row in table.get():
sm += row[column]
return sm
def average(self, table, column):
coltp = type(next(table.get())[column])
if coltp != int and coltp != float:
raise FlowError('Cannot apply aggregate to non-numerical' +
' field.')
sm = 0.0
for row in table.get():
sm += row[column]
return sm / table.size()
self.logic = AggregateLogic
self.exec_args = (self.aggregate, self.column)
def optimize(flow, rules: dict = DEFAULT_RULES):
for key in DEFAULT_RULES:
if key not in rules:
rules[key] = False
if rules['colocate'] and rules['breakpoint']:
raise FlowError('Cannot enable the colocate and breakpoint rules' +
' together.')
optimized = Flow(flow.flowname, flow.typ, flow.cloudburst, flow.source)
if rules['whole']:
cloned = optimize(
flow, {
'fusion': False,
'compete': False,
'compete_replicas': 1,
'colocate': False,
'breakpoint': False,
'whole': False
})
cloned.cloudburst = None # Remove sockets to serialize and send flow.
queue = [cloned]
gpu = False
batching = []
while len(queue) > 0:
op = queue.pop(0)
op.cb_fn = None
if type(op) != Flow:
batching.append(op.batching)
gpu = op.gpu if not gpu else gpu
queue.extend(op.downstreams)
if all(batching):
cloned.batching = True
optimized.multi([cloned], whole=True)
multi_op = optimized.downstreams[0]
multi_op.batching = all(batching)
multi_op.gpu = gpu
if gpu:
multi_op.fn_name += '-gpu'
return optimized
### OPERATOR FUSION ###
queue = []
join_tracker = {}
processed = set()
for ds in flow.downstreams:
queue.append((ds, optimized))
# NOTE: We clone the whole flow regardless. If fusion is turned on,
# then we will fuse operators, and otherwise, we simply find chains,
# throw them away, and add operators to the optimized flow.
while len(queue) > 0:
op, upstream = queue.pop(0)
if op.fn_name in processed:
continue
chain = find_chain(op)
if len(chain) == 0 or not rules['fusion']:
downstreams = op.downstreams
processed.add(op.fn_name)
if type(op) == MapOperator:
marker = upstream.map(op.fn, op.col, op.names,
op.logic.preprocess, op.high_variance,
op.gpu, op.batching, op.multi)
if type(op) == FilterOperator:
marker = upstream.filter(op.fn, op.group, op.logic.preprocess)
if type(op) == GroupbyOperator:
marker = upstream.gropuby(op.groupby_key, op.logic.preprocess)
if type(op) == CombineOperator:
marker = upstream.combine()
if type(op) == LookupOperator:
# Merge lookup operators with their successors.
downstreams = []
for ds in op.downstreams:
if isinstance(ds, MultiOperator):
ops = [op] + ds.ops
else:
ops = [op, ds]
marker = upstream.multi(ops)
for next_ds in ds.downstreams:
queue.append((next_ds, marker))
if type(op) == AggOperator:
marker = upstream.agg(op.aggregate, op.column)
if type(op) == MultiOperator:
# This will only happen in the case where the previous operator
# was a LookupHelperOperator combined with something else.
marker = upstream.multi(op.ops)
if type(op) == JoinOperator:
if op.fn_name not in join_tracker:
join_tracker[op.fn_name] = upstream
downstreams = []
processed.discard(op.fn_name)
else:
other = join_tracker[op.fn_name]
marker = other.join(upstream, op.on, op.how,
op.logic.preprocess)
else:
marker = upstream.multi(chain)
downstreams = chain[-1].downstreams
for op in chain:
# Set the multi operator to have various properties.
if op.high_variance:
optimized.operators[marker.position].high_variance = True
if op.gpu:
optimized.operators[marker.position].gpu = True
# Hack for autoscaling...
optimized.operators[marker.position].fn_name += '-gpu'
if op.batching:
optimized.operators[marker.position].batching = True
if optimized.operators[marker.position].batching:
for old in chain:
if not old.batching:
print('Cannot create a fused operator with' +
' batching enabled if all operators do' +
' not batch.')
optimized.operators[marker.position].batching = False
for ds in downstreams:
queue.append((ds, marker))
### LOCALITY BREAKPOINTS ###
if rules['breakpoint']:
queue = [optimized]
processed = set()
while len(queue) > 0:
op = queue.pop(0)
if op.fn_name in processed:
continue
# We only set breakpoints if we are in a linear chain portion of the
# flow. This will only be true if there is only one operator in the
# queue at a time. After pop, the length should be 0 until we add this
# op's downstreams.
if len(queue) == 0:
if isinstance(op, LookupOperator):
op.breakpoint = True
if isinstance(op, MultiOperator):
for sub in op.ops:
if isinstance(sub, LookupOperator):
op.breakpoint = True
processed.add(op.fn_name)
queue.extend(op.downstreams)
### COMPETITIVE EXECUTION ###
if rules['compete']:
new_ops = []
for operator in optimized.operators.values():
if operator.high_variance:
for downstream in operator.downstreams:
if len(downstream.upstreams) > 1:
raise RuntimeError("Cannot have a competitive" +
" execution map feed into an " +
"operator with multiple upstreams.")
downstream.multi_exec = True
for _ in range(rules['compete_replicas']):
# Create a new operator that is an exact replica.
if isinstance(operator, MapOperator):
new_op = MapOperator(operator.fn, operator.fntype,
operator.flowname, operator.col,
operator.names,
operator.logic.preprocess,
operator.high_variance,
operator.gpu, operator.batching,
operator.multi, optimized.sink)
if isinstance(operator, MultiOperator):
new_op = MultiOperator(operator.ops, operator.flowname,
optimized.sink)
# Hook it into the DAG by updating all up/downstreams.
new_op.downstreams = list(operator.downstreams)
new_op.upstreams = list(operator.upstreams)
for op in new_op.downstreams:
op.upstreams.append(new_op)
for op in new_op.upstreams:
op.downstreams.append(new_op)
new_ops.append(new_op)
for new_op in new_ops:
optimized.operators[str(uuid.uuid4())] = new_op
if rules['colocate']:
curr_op = optimized
while len(curr_op.downstreams) > 0:
if len(curr_op.downstreams) == 1:
curr_op = curr_op.downstreams[0]
else: # We only support one colocation for now.
if not curr_op.supports_broadcast:
raise RuntimeError('Unsupported broadcast attempt.')
colocates = list(
map(lambda op: op.fn_name, curr_op.downstreams))
optimized.colocates = colocates
for op in curr_op.downstreams:
if not curr_op.supports_broadcast:
raise RuntimeError('Unsupported broadcast attempt.')
args = list(op.init_args)
args[1] = True # Receive broadcast.
op.init_args = tuple(args)
args = list(curr_op.init_args)
args[0] = True # Send broadcast.
curr_op.init_args = tuple(args)
break
return optimized
def extend(self, other, marker: FlowMarker = None):
assert isinstance(other, Flow)
# Check the current flow to make sure it has only one final operator.
queue = [self]
seen = set()
num_ends = 0
sinks = []
join_tracker = {}
while len(queue) > 0:
op = queue.pop(0)
if op.fn_name not in seen and len(op.downstreams) == 0:
num_ends += 1
seen.add(op.fn_name)
queue.extend(op.downstreams)
# If there are multiple operators with 0 downstreams, raise an error.
if num_ends > 1:
raise FlowError(
'Cannot extend a flow when there are multiple'
' unmerged operators. Flow can only have one sink.')
# Iterate over everything in the
queue = list(other.downstreams)
upstream_map = {}
seen = set()
for op in other.downstreams:
upstream_map[op.fn_name] = marker if marker else self
while len(queue) > 0:
op = queue.pop(0)
if op.fn_name in seen:
continue
seen.add(op.fn_name)
next_marker = upstream_map[op.fn_name]
if type(op) == MapOperator:
marker = next_marker.map(op.fn, op.col, op.names,
op.logic.preprocess, op.high_variance,
op.gpu, op.batching, op.multi)
if type(op) == FilterOperator:
marker = next_marker.filter(op.fn, op.group,
op.logic.preprocess)
if type(op) == GroupbyOperator:
marker = next_marker.gropuby(op.groupby_key,
op.logic.preprocess)
if type(op) == CombineOperator:
marker = next_marker.combine()
if type(op) == LookupOperator:
# Merge lookup operators with their successors.
marker = op.lookup(op.lookup_key, op.dynamic, op.dummy)
if type(op) == AggOperator:
marker = upstream.agg(op.aggregate, op.column)
if type(op) == MultiOperator:
# This will only happen in the case where the previous operator
# was a LookupHelperOperator combined with something else.
marker = upstream.multi(op.ops)
if type(op) == JoinOperator:
if op.fn_name not in join_tracker:
join_tracker[op.fn_name] = upstream
downstreams = []
processed.discard(op.fn_name)
else:
other = join_tracker[op.fn_name]
marker = other.join(upstream, op.on, op.how,
op.logic.preprocess)
for ds in op.downstreams:
upstream_map[ds.fn_name] = marker
queue.extend(op.downstreams)
def deploy(self, cloudburst):
if self.deployed:
return [], [], {}, [self.fn_name], {}, [], []
if not self.registered:
self.register(cloudburst)
functions, connections, gpus, batching = [], [], [], []
# NOTE: What if we need something other than None here? Will need an
# arg at some point, maybe. This is for multi-execution pick-first
# result back functions.
if self.multi_exec:
functions.append((self.fn_name, [None]))
else:
functions.append(self.fn_name)
if self.gpu:
gpus.append(self.fn_name)
if self.batching:
batching.append(self.fn_name)
arg_map = {}
arg_map[self.fn_name] = list(self.get_exec_args())
registered = {}
for ds in self.downstreams:
fn_names, ds_conns, fn_args, starts, ds_registered, ds_gpus, \
ds_batching = \
ds.deploy(cloudburst)
for fn_name in starts:
connections.append((self.fn_name, fn_name))
functions += fn_names
connections += ds_conns
gpus += ds_gpus
batching += ds_batching
arg_map.update(fn_args)
registered.update(ds_registered)
self.deployed = True
# We can number these linearly because the optimization algorithm
# guarantees that breakpoints will only be at linear points in the
# flow.
if self.breakpoint:
uid = str(len(registered))
name = self.flowname + '-' + uid
success, error = cloudburst.register_dag(
name,
functions,
connections,
gpu_functions=gpus,
batching_functions=batching)
if not success:
raise FlowError(str(error))
registered[name] = arg_map
return [], [], {}, [], registered, [], []
else:
return functions, connections, arg_map, [self.fn_name], \
registered, gpus, batching
def __init__(self,
fn: Callable,
fntype: FunctionType,
flowname: str,
col: str,
names: List[str],
init: Callable,
high_variance: bool,
gpu: bool,
batching: bool,
multi: bool,
sink: list):
self.__name__ = 'MapOperator'
self.fn = fn
self._setup(flowname)
self.fntype = fntype
self.sink = sink
self.col = col
self.names = names
self.high_variance = high_variance
self.gpu = gpu
self.batching = batching
self.multi = multi
self.supports_broadcast = True
if col is not None: # Map over column not over whole row.
if len(names) >= 2: # We can only rename one column here
raise FlowError('Map over a column cannot rename multiple'
+ ' columns.')
if gpu:
raise FlowError('You cannot do a column map with batching.')
else:
if len(names) != 0 and len(names) != len(fntype.ret):
raise FlowError('Map over row must have same number of columns'
+ ' as function outputs.')
class MapLogic:
def __init__(self, cloudburst, send_broadcast, recv_broadcast,
batching, multi):
# We pass in None in local mode because we expect the
# Cloudburst user library in the other case.
self.send_broadcast = send_broadcast
self.recv_broadcast = recv_broadcast
self.batching = batching
self.multi = multi
self.preprocess(cloudburst)
def preprocess(self, _):
pass
def run(self, cloudburst, fn, fntype, col, names, inp):
# Merge all of the tables.
serialized = False
batching = self.batching and isinstance(inp, list)
if batching:
if type(inp[0]) == bytes:
inp = [deserialize(tbl) for tbl in inp]
serialized = True
# inp will be a list of Tables. If it not, this is part of
# a MultiOperator, and everything is taken care of for us.
merged, mappings = merge_tables(inp)
inp = merged
# This will all be repeated because of the way Cloudburst's
# batching works, so we just pick the first one. But we
# check because even with batching enabled, in a multi
# operator, we will not have to deal with this.
if type(fn) == list:
fn = fn[0]
if type(fntype) == list:
fntype = fntype[0]
if type(col) == list:
col = col[0]
if type(names) == list and type(names[0]) == list:
names = names[0]
else:
if type(inp) == bytes:
inp = deserialize(inp)
serialized = True
schema = []
if col is None:
if len(names) != 0:
schema = list(zip(names, fntype.ret))
else:
for i in range(len(fntype.ret)):
schema.append((str(i), fntype.ret[i]))
else:
for name, tp in inp.schema:
if name != col:
schema.append((name, tp))
else:
if len(names) != 0:
schema.append((names[0], fntype.ret[0]))
else:
schema.append((name, fntype.ret[0]))
if isinstance(inp, GroupbyTable):
result = GroupbyTable(schema, inp.col)
for group, gtable in inp.get():
result.add_group(group, self.run(fn, fntype, col, gtable))
else:
result = Table(schema)
if self.batching or self.multi:
res = fn(self, inp)
for val in res:
if type(val) == tuple:
val = list(val)
elif type(val) != list:
val = [val]
result.insert(val)
else:
for row in inp.get():
if col is None:
vals = fn(self, row)
if type(vals) == tuple:
vals = list(vals)
elif type(vals) != list:
vals = [vals]
result.insert(vals, row[Row.qid_key])
else:
val = fn(self, row[col])
new_vals = []
for name, _ in inp.schema:
if name == col:
new_vals.append(val)
else:
new_vals.append(row[name])
result.insert(new_vals, row[Row.qid_key])
if batching: # Unmerge all the tables.
tables = demux_tables(result, mappings)
result = tables
if serialized:
result = [serialize(tbl) for tbl in result]
else:
if serialized:
result = serialize(result)
if self.send_broadcast:
import uuid
uid = str(uuid.uuid4())
cloudburst.put(uid, result)
result = uid
return result
self.logic = MapLogic
if init is not None:
self.logic.preprocess = init
self.exec_args = (self.fn, self.fntype, self.col, self.names)
self.init_args = (False, False, self.batching, self.multi)
