def _rewrite_hybrid_join(self, node: saldag.HybridJoin):
# TODO
suffix = "rand"
# in left parents' children, replace self with first primitive operator
# in expanded subdag
shuffled_a = sal.shuffle(node.left_parent, "shuffled_a")
shuffled_a.is_mpc = True
node.left_parent.children.remove(node)
# same for right parent
shuffled_b = sal.shuffle(node.right_parent, "shuffled_b")
shuffled_b.is_mpc = True
node.right_parent.children.remove(node)
persisted_b = sal._persist(shuffled_b, "persisted_b")
persisted_b.is_mpc = True
persisted_a = sal._persist(shuffled_a, "persisted_a")
persisted_a.is_mpc = True
keys_a_closed = sal.project(shuffled_a, "keys_a_closed", ["a"])
keys_a_closed.is_mpc = True
keys_b_closed = sal.project(shuffled_b, "keys_b_closed", ["c"])
keys_b_closed.is_mpc = True
keys_a = sal._open(keys_a_closed, "keys_a", 1)
keys_a.is_mpc = True
keys_b = sal._open(keys_b_closed, "keys_b", 1)
keys_b.is_mpc = True
indexed_a = sal.index(keys_a, "indexed_a", "index_a")
indexed_a.is_mpc = False
indexed_b = sal.index(keys_b, "indexed_b", "index_b")
indexed_b.is_mpc = False
joined_indices = sal.join(indexed_a, indexed_b, "joined_indices",
["a"], ["c"])
joined_indices.is_mpc = False
indices_only = sal.project(joined_indices, "indices_only",
["index_a", "index_b"])
indices_only.is_mpc = False
# TODO: update stored_with to use union of parent out_rel stored_with sets
indices_closed = sal._close(indices_only, "indices_closed", set([1,
2]))
indices_closed.is_mpc = True
joined = sal._index_join(persisted_a, persisted_b, "joined", ["a"],
["c"], indices_closed)
joined.is_mpc = True
# replace self with leaf of expanded subdag in each child node
for child in node.get_sorted_children():
child.replace_parent(node, joined)
# add former children to children of leaf
joined.children = node.children
def hybrid_agg(in1):
shuffled = sal.shuffle(in1, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["b"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
distinctKeys = sal.distinct(keys, "distinctKeys", ["b"])
distinctKeys.isMPC = False
distinctKeys.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
fakeDistinctKeys = sal.project(distinctKeys, "distinctKeys", ["b"])
fakeDistinctKeys.isMPC = False
fakeDistinctKeys.out_rel.storedWith = set([1])
indexedDistinct = sal.index(distinctKeys, "indexedDistinct",
"keyIndex")
indexedDistinct.isMPC = False
indexedDistinct.out_rel.storedWith = set([1])
joinedindeces = sal.join(indexed, indexedDistinct, "joinedindeces",
["b"], ["b"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
# TODO: could project row indeces away too
indecesonly = sal.project(joinedindeces, "indecesonly",
["rowIndex", "keyIndex"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
closedDistinct = sal._close(distinctKeys, "closedDistinct",
set([1, 2, 3]))
closedDistinct.isMPC = True
closedLookup = sal._close(indecesonly, "closedLookup", set([1, 2, 3]))
closedLookup.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["b"], "d", "+", "d",
closedLookup, closedDistinct)
agg.isMPC = True
sal._open(agg, "aggopened", 1)
def protocol():
inpts = setup()
in_1 = inpts[0]
ind = sal.index(in_1, "index_1", "index")
out = sal.collect(ind, 1)
return set([in_1])
def protocol():
inpts = setup()
in_1 = inpts[0]
ind = sal.index(in_1, "ind")
out = sal.collect(ind, 1)
return {in_1}
def protocol():
inpts = setup()
in_1 = inpts[0]
ind = cc.index(in_1, "index_1", "index")
out = cc.collect(ind, 1)
return {in_1}
def protocol():
inpts = setup()
in_1 = inpts
indexed = sal.index(in_1, 'index_1', 'index')
out = sal.collect(indexed, 1)
return set([in_1])
def hybrid_agg(in1):
shuffled = sal.shuffle(in1, "shuffled")
shuffled.out_rel.stored_with = set([1, 2, 3])
shuffled.is_mpc = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.stored_with = set([1, 2, 3])
persisted.is_mpc = True
keys_closed = sal.project(shuffled, "keys_closed", ["b"])
keys_closed.out_rel.stored_with = set([1, 2, 3])
keys_closed.is_mpc = True
keys = sal._open(keys_closed, "keys", 1)
keys.is_mpc = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.is_mpc = False
indexed.out_rel.stored_with = set([1])
sorted_by_key = sal.sort_by(indexed, "sorted_by_key", "b")
sorted_by_key.is_mpc = False
sorted_by_key.out_rel.stored_with = set([1])
eq_flags = sal._comp_neighs(sorted_by_key, "eq_flags", "b")
eq_flags.is_mpc = False
eq_flags.out_rel.stored_with = set([1])
# TODO: should be a persist op
sorted_by_key_stored = sal.project(
sorted_by_key, "sorted_by_key_stored", ["rowIndex", "b"])
sorted_by_key_stored.is_mpc = False
sorted_by_key_stored.out_rel.stored_with = set([1])
closed_eq_flags = sal._close(eq_flags, "closed_eq_flags", set([1, 2, 3]))
closed_eq_flags.is_mpc = True
closed_sorted_by_key = sal._close(
sorted_by_key_stored, "closed_sorted_by_key", set([1, 2, 3]))
closed_sorted_by_key.is_mpc = True
agg = sal.index_aggregate(
persisted, "agg", ["b"], "d", "+", "d", closed_eq_flags, closed_sorted_by_key)
agg.is_mpc = True
sal._open(agg, "ssnopened", 1)
def hybrid_agg(in1):
shuffled = sal.shuffle(in1, "shuffled")
shuffled.out_rel.storedWith = {1, 2, 3}
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = {1, 2, 3}
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["b"])
keysclosed.out_rel.storedWith = {1, 2, 3}
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = {1}
sortedByKey = sal.sort_by(indexed, "sortedByKey", "b")
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = {1}
eqFlags = sal._comp_neighs(sortedByKey, "eqFlags", "b")
eqFlags.isMPC = False
eqFlags.out_rel.storedWith = {1}
# TODO: should be a persist op
sortedByKeyStored = sal.project(sortedByKey, "sortedByKeyStored",
["rowIndex", "b"])
sortedByKeyStored.isMPC = False
sortedByKeyStored.out_rel.storedWith = {1}
closedEqFlags = sal._close(eqFlags, "closedEqFlags", {1, 2, 3})
closedEqFlags.isMPC = True
closedSortedByKey = sal._close(sortedByKeyStored, "closedSortedByKey",
{1, 2, 3})
closedSortedByKey.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["b"], "d", "+", "d",
closedEqFlags, closedSortedByKey)
agg.isMPC = True
sal._open(agg, "ssnopened", 1)
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
# define inputs
colsInB = [
defCol("a", "INTEGER", [2]),
defCol("b", "INTEGER", [2]),
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["a", "b"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
# define inputs
colsInC = [
defCol("a", "INTEGER", [3]),
defCol("b", "INTEGER", [3]),
]
in3 = sal.create("in3", colsInC, set([3]))
in3.isMPC = False
projc = sal.project(in3, "projc", ["a", "b"])
projc.isMPC = False
projc.out_rel.storedWith = set([3])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
clC = sal._close(projc, "clC", set([1, 2, 3]))
clC.isMPC = True
comb = sal.concat([clA, clB, clC], "comb")
comb.out_rel.storedWith = set([1, 2, 3])
comb.isMPC = True
shuffled = sal.shuffle(comb, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["a"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
sortedByKey = sal.sort_by(indexed, "sortedByKey", "a")
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
eqFlags = sal._comp_neighs(sortedByKey, "eqFlags", "a")
eqFlags.isMPC = False
eqFlags.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
sortedByKey = sal.project(sortedByKey, "sortedByKey",
["rowIndex", "a"])
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
closedEqFlags = sal._close(eqFlags, "closedEqFlags", set([1, 2, 3]))
closedEqFlags.isMPC = True
closedSortedByKey = sal._close(sortedByKey, "closedSortedByKey",
set([1, 2, 3]))
closedSortedByKey.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["a"], "b", "+", "b",
closedEqFlags, closedSortedByKey)
agg.out_rel.storedWith = set([1, 2, 3])
agg.isMPC = True
sal._open(agg, "opened", 1)
# create condag
return set([in1, in2, in3])
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
persistedA = sal._persist(clA, "persistedA")
persistedB = sal._persist(clB, "persistedB")
keysaclosed = sal.project(clA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(clB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined",
["a"], ["c"], indecesclosed)
joined.isMPC = True
sal._open(joined, "opened", 1)
# create condag
return set([in1, in2])
def hybrid_join():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("govreg", colsInA, {1})
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = {1}
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("company0", colsInB, {2})
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = {2}
colsInC = [
defCol("c", "INTEGER", [1], [3]),
defCol("d", "INTEGER", [3])
]
in3 = sal.create("company1", colsInC, {3})
in3.isMPC = False
projc = sal.project(in3, "projc", ["c", "d"])
projc.isMPC = False
projc.out_rel.storedWith = {3}
clA = sal._close(proja, "clA", {1, 2, 3})
clA.isMPC = True
clB = sal._close(projb, "clB", {1, 2, 3})
clB.isMPC = True
clC = sal._close(projc, "clC", {1, 2, 3})
clC.isMPC = True
rightClosed = sal.concat([clB, clC], "clD")
rightClosed.isMPC = True
rightClosed.out_rel.storedWith = {1, 2, 3}
shuffledA = sal.shuffle(clA, "shuffledA")
shuffledA.isMPC = True
persistedA = sal._persist(shuffledA, "persistedA")
persistedA.isMPC = True
shuffledB = sal.shuffle(rightClosed, "shuffledB")
shuffledB.isMPC = True
persistedB = sal._persist(shuffledB, "persistedB")
persistedB.isMPC = True
keysaclosed = sal.project(shuffledA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = {1, 2, 3}
keysaclosed.isMPC = True
keysbclosed = sal.project(shuffledB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = {1, 2, 3}
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = {1}
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = {1}
joinedindeces = sal.join(indexedA, indexedB, "joinedindeces", ["a"],
["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = {1}
indecesonly = sal.project(joinedindeces, "indecesonly",
["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = {1}
indecesclosed = sal._close(indecesonly, "indecesclosed", {1, 2, 3})
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined", ["a"],
["c"], indecesclosed)
joined.isMPC = True
return joined, {in1, in2, in3}
def hybrid_join():
# define inputs
cols_in_a = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in_1 = sal.create("govreg", cols_in_a, set([1]))
in_1.is_mpc = False
proj_a = sal.project(in_1, "proj_a", ["a", "b"])
proj_a.is_mpc = False
proj_a.out_rel.stored_with = set([1])
cols_in_b = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in_2 = sal.create("company0", cols_in_b, set([2]))
in_2.is_mpc = False
proj_b = sal.project(in_2, "proj_b", ["c", "d"])
proj_b.is_mpc = False
proj_b.out_rel.stored_with = set([2])
cols_in_c = [
defCol("c", "INTEGER", [1], [3]),
defCol("d", "INTEGER", [3])
]
in_3 = sal.create("company1", cols_in_c, set([3]))
in_3.is_mpc = False
proj_c = sal.project(in_3, "proj_c", ["c", "d"])
proj_c.is_mpc = False
proj_c.out_rel.stored_with = set([3])
cl_a = sal._close(proj_a, "cl_a", set([1, 2, 3]))
cl_a.is_mpc = True
cl_b = sal._close(proj_b, "cl_b", set([1, 2, 3]))
cl_b.is_mpc = True
cl_c = sal._close(proj_c, "cl_c", set([1, 2, 3]))
cl_c.is_mpc = True
right_closed = sal.concat([cl_b, cl_c], "clD")
right_closed.is_mpc = True
right_closed.out_rel.stored_with = set([1, 2, 3])
shuffled_a = sal.shuffle(cl_a, "shuffled_a")
shuffled_a.is_mpc = True
persisted_a = sal._persist(shuffled_a, "persisted_a")
persisted_a.is_mpc = True
shuffled_b = sal.shuffle(right_closed, "shuffled_b")
shuffled_b.is_mpc = True
persisted_b = sal._persist(shuffled_b, "persisted_b")
persisted_b.is_mpc = True
keys_a_closed = sal.project(shuffled_a, "keys_a_closed", ["a"])
keys_a_closed.out_rel.stored_with = set([1, 2, 3])
keys_a_closed.is_mpc = True
keys_b_closed = sal.project(shuffled_b, "keys_b_closed", ["c"])
keys_b_closed.is_mpc = True
keys_b_closed.out_rel.stored_with = set([1, 2, 3])
keys_a = sal._open(keys_a_closed, "keys_a", 1)
keys_a.is_mpc = True
keys_b = sal._open(keys_b_closed, "keys_b", 1)
keys_b.is_mpc = True
indexed_a = sal.index(keys_a, "indexed_a", "index_a")
indexed_a.is_mpc = False
indexed_a.out_rel.stored_with = set([1])
indexed_b = sal.index(keys_b, "indexed_b", "index_b")
indexed_b.is_mpc = False
indexed_b.out_rel.stored_with = set([1])
joined_indeces = sal.join(
indexed_a, indexed_b, "joined_indeces", ["a"], ["c"])
joined_indeces.is_mpc = False
joined_indeces.out_rel.stored_with = set([1])
indeces_only = sal.project(
joined_indeces, "indeces_only", ["index_a", "index_b"])
indeces_only.is_mpc = False
indeces_only.out_rel.stored_with = set([1])
indeces_closed = sal._close(
indeces_only, "indeces_closed", set([1, 2, 3]))
indeces_closed.is_mpc = True
joined = sal._index_join(persisted_a, persisted_b, "joined", [
"a"], ["c"], indeces_closed)
joined.is_mpc = True
return joined, set([in_1, in_2, in_3])
