def project(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
defCol('c', 'INTEGER', [1]),
defCol('d', 'INTEGER', [1])
]
in1 = sal.create("in1", colsInA, set([1]))
cols = ([column.name for column in in1.out_rel.columns])
shuffle(cols)
proja = sal.project(in1, "proja", cols)
return set([in1])
dag = protocol()
config = CodeGenConfig('project_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/project_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def scalar_div(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
div1 = sal.divide(in1, 'div1', 'a', ['a', 5])
return set([in1])
dag = protocol()
config = CodeGenConfig('scalar_div_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/scalar_div_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def dispatch_jobs(job_queue: list, conclave_config: CodeGenConfig, time_dispatch: bool = False):
"""
Dispatches jobs to respective backends.
:param time_dispatch: will record the execution time of dispatch if true
:param job_queue: jobs to dispatch
:param conclave_config: conclave configuration
"""
networked_peer = None
# if more than one party is involved in the protocol, we need a networked peer
if len(conclave_config.all_pids) > 1:
networked_peer = _setup_networked_peer(conclave_config.network_config)
if time_dispatch:
# TODO use timeit
import time
import datetime
start_time = time.time()
dispatch_all(conclave_config, networked_peer, job_queue)
elapsed_time = time.time() - start_time
formatted_time = datetime.timedelta(milliseconds=(elapsed_time * 1000))
print("TIMED", conclave_config.name, round(elapsed_time, 3), formatted_time)
with open("timing_results.csv", "a+") as time_f:
out = ",".join([conclave_config.name, str(round(elapsed_time, 3)), str(formatted_time)])
time_f.write(out + "\n")
else:
dispatch_all(conclave_config, networked_peer, job_queue)
def join(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
]
colsInB = [
defCol('a', 'INTEGER', [1]),
defCol('c', 'INTEGER', [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in2 = sal.create("in2", colsInB, set([1]))
join1 = sal.join(in1, in2, 'join1', ['a'], ['a'])
return set([in1, in2])
dag = protocol()
config = CodeGenConfig('join_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/join_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def dispatch_jobs(job_queue: list, conclave_config: CodeGenConfig):
"""
Dispatches jobs to respective backends.
:param job_queue: jobs to dispatch
:param conclave_config: conclave configuration
"""
networked_peer = None
# if more than one party is involved in the protocol, we need a networked peer
if len(conclave_config.all_pids) > 1:
networked_peer = _setup_networked_peer(conclave_config.network_config)
dispatch_all(conclave_config, networked_peer, job_queue)
def testHybridAggWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
# define inputs
colsInB = [
defCol("a", "INTEGER", [2]),
defCol("b", "INTEGER", [2]),
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["a", "b"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
# define inputs
colsInC = [
defCol("a", "INTEGER", [3]),
defCol("b", "INTEGER", [3]),
]
in3 = sal.create("in3", colsInC, set([3]))
in3.isMPC = False
projc = sal.project(in3, "projc", ["a", "b"])
projc.isMPC = False
projc.out_rel.storedWith = set([3])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
clC = sal._close(projc, "clC", set([1, 2, 3]))
clC.isMPC = True
comb = sal.concat([clA, clB, clC], "comb")
comb.out_rel.storedWith = set([1, 2, 3])
comb.isMPC = True
shuffled = sal.shuffle(comb, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["a"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
sortedByKey = sal.sort_by(indexed, "sortedByKey", "a")
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
eqFlags = sal._comp_neighs(sortedByKey, "eqFlags", "a")
eqFlags.isMPC = False
eqFlags.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
sortedByKey = sal.project(sortedByKey, "sortedByKey",
["rowIndex", "a"])
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
closedEqFlags = sal._close(eqFlags, "closedEqFlags", set([1, 2, 3]))
closedEqFlags.isMPC = True
closedSortedByKey = sal._close(sortedByKey, "closedSortedByKey",
set([1, 2, 3]))
closedSortedByKey.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["a"], "b", "+", "b",
closedEqFlags, closedSortedByKey)
agg.out_rel.storedWith = set([1, 2, 3])
agg.isMPC = True
sal._open(agg, "opened", 1)
# create condag
return set([in1, in2, in3])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "hybrid-agg-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False,
use_docker=True)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/" + size
codegen_config.output_path = "/mnt/shared/" + size
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag,
pid).generate("sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config,
subdag).generate("python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = {
"pid": pid,
"parties": {
1: {
"host": "ca-spark-node-0",
"port": 9001
},
2: {
"host": "cb-spark-node-0",
"port": 9002
},
3: {
"host": "cc-spark-node-0",
"port": 9003
}
}
}
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
def testPublicJoinWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
persistedA = sal._persist(clA, "persistedA")
persistedB = sal._persist(clB, "persistedB")
keysaclosed = sal.project(clA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(clB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined",
["a"], ["c"], indecesclosed)
joined.isMPC = True
sal._open(joined, "opened", 1)
# create condag
return set([in1, in2])
pid = int(sys.argv[1])
workflow_name = "hybrid-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared"
codegen_config.output_path = "/mnt/shared"
exampleutils.generate_data(pid, codegen_config.output_path)
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = exampleutils.get_sharemind_config(pid, True)
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
if pid == 1:
expected = ['', '2,200,2001', '3,300,3001', '4,400,4001', '42,42,1001', '5,500,5001',
'6,600,6001', '7,700,7001', '7,800,7001', '7,900,7001', '8,1000,8001', '9,1100,9001']
exampleutils.check_res(expected, "/mnt/shared/opened.csv")
print("Success")
def testPublicJoinWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
persistedA = sal._persist(clA, "persistedA")
persistedA.isMPC = True
persistedB = sal._persist(clB, "persistedB")
persistedB.isMPC = True
keysaclosed = sal.project(clA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(clB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined", [
"a"], ["c"], indecesclosed)
joined.out_rel.storedWith = set([1, 2, 3])
joined.isMPC = True
sal._open(joined, "opened", 1)
# create condag
return set([in1, in2])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "public-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False, use_docker=True)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/hybridjoin/" + size
codegen_config.output_path = "/mnt/shared/hybridjoin/" + size
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = {
"pid": pid,
"parties": {
1: {"host": "ca-spark-node-0", "port": 9001},
2: {"host": "cb-spark-node-0", "port": 9002},
3: {"host": "cc-spark-node-0", "port": 9003}
}
}
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
def testHybridJoinWorkflow():
def hybrid_join():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
shuffledA = sal.shuffle(clA, "shuffledA")
shuffledA.isMPC = True
persistedA = sal._persist(shuffledA, "persistedA")
persistedA.isMPC = True
shuffledB = sal.shuffle(clB, "shuffledB")
shuffledB.isMPC = True
persistedB = sal._persist(shuffledB, "persistedB")
persistedB.isMPC = True
keysaclosed = sal.project(shuffledA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(shuffledB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined", [
"a"], ["c"], indecesclosed)
joined.isMPC = True
return joined, set([in1, in2])
def hybrid_agg(in1):
shuffled = sal.shuffle(in1, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["b"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
distinctKeys = sal.distinct(keys, "distinctKeys", ["b"])
distinctKeys.isMPC = False
distinctKeys.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
fakeDistinctKeys = sal.distinct(keys, "distinctKeys", ["b"])
fakeDistinctKeys.isMPC = False
fakeDistinctKeys.out_rel.storedWith = set([1])
indexedDistinct = sal.index(distinctKeys, "indexedDistinct", "keyIndex")
indexedDistinct.isMPC = False
indexedDistinct.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexed, indexedDistinct, "joinedindeces", ["b"], ["b"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
# TODO: could project row indeces away too
indecesonly = sal.project(
joinedindeces, "indecesonly", ["rowIndex", "keyIndex"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
closedDistinct = sal._close(distinctKeys, "closedDistinct", set([1, 2, 3]))
closedDistinct.isMPC = True
closedLookup = sal._close(indecesonly, "closedLookup", set([1, 2, 3]))
closedLookup.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["b"], "d", "+", "d", closedLookup, closedDistinct)
agg.isMPC = True
sal._open(agg, "aggopened", 1)
def protocol():
joinedres, inputs = hybrid_join()
hybrid_agg(joinedres)
return saldag.OpDag(inputs)
pid = int(sys.argv[1])
workflow_name = "ssn-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False, use_docker=False)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared"
codegen_config.output_path = "/mnt/shared"
exampleutils.generate_ssn_data(pid, codegen_config.output_path)
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = exampleutils.get_sharemind_config(pid, True)
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
if pid == 1:
expected = ['', '1,30', '2,50', '3,30']
exampleutils.check_res(expected, "/mnt/shared/aggopened.csv")
print("Success")