def local_main():
current_dir = os.path.dirname(os.path.realpath(__file__))
data_path = os.path.join(current_dir, "data")
for pid in {"1", "2"}:
# define name for the workflow
workflow_name = "aspirin-local-test-" + pid
# configure conclave
conclave_config = CodeGenConfig(workflow_name, int(pid))
conclave_config.all_pids = [int(pid)]
sharemind_conf = SharemindCodeGenConfig("/mnt/shared", use_docker=False, use_hdfs=False)
conclave_config.with_sharemind_config(sharemind_conf)
# point conclave to the directory where the generated code should be stored/ read from
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
# point conclave to directory where data is to be read from...
conclave_config.input_path = data_path
# and written to
conclave_config.output_path = data_path
suffix = "left" if pid == "1" else "right"
# define this party's unique ID (in this demo there is only one party)
job_queue = generate_code(lambda: protocol_local(suffix, int(pid)), conclave_config, ["sharemind"], ["python"],
apply_optimizations=False)
dispatch_jobs(job_queue, conclave_config)
res_mpc = read_rel(data_path + "/" + "actual_mpc_open.csv")
res_left = read_rel(data_path + "/" + "actual_left.csv")
res_right = read_rel(data_path + "/" + "actual_right.csv")
assert len(res_mpc) == 1
assert len(res_left) == 1
assert len(res_right) == 1
res = [[res_mpc[0][0] + res_left[0][0] + res_right[0][0]]]
write_rel(data_path, "actual_open.csv", res, "1")
def run_mpc(pid: str, data_root: str, mpc_backend: str):
workflow_name = "aspirin-mpc-join-" + pid + "-" + data_root
conclave_config = CodeGenConfig(workflow_name, int(pid))
conclave_config.use_leaky_ops = False
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
conclave_config.input_path = os.path.join("/mnt/shared", data_root)
conclave_config.output_path = os.path.join("/mnt/shared", data_root)
job_queue = generate_code(lambda: protocol_mpc(conclave_config.all_pids),
conclave_config, [mpc_backend], ["python"],
apply_optimizations=True)
dispatch_jobs(job_queue, conclave_config)
def main_mpc(pid: str, mpc_backend: str):
# define name for the workflow
workflow_name = "real-aspirin-partitioned-" + pid
# configure conclave
mpc_backend = sys.argv[2]
conclave_config = CodeGenConfig(workflow_name, int(pid)) \
.with_default_mpc_config(mpc_backend)
current_dir = os.path.dirname(os.path.realpath(__file__))
# point conclave to the directory where the generated code should be stored/ read from
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
# point conclave to directory where data is to be read from...
conclave_config.input_path = os.path.join(current_dir, "data")
# and written to
conclave_config.output_path = os.path.join(current_dir, "data")
job_queue = generate_code(lambda: protocol_mpc(conclave_config.all_pids), conclave_config, [mpc_backend],
["python"], apply_optimizations=True)
dispatch_jobs(job_queue, conclave_config)
def run_local(pid: str, data_root: str):
workflow_name = "aspirin-local-join-" + pid + "-" + data_root
conclave_config = CodeGenConfig(workflow_name, int(pid))
conclave_config.all_pids = [int(pid)]
sharemind_conf = SharemindCodeGenConfig("/mnt/shared",
use_docker=False,
use_hdfs=False)
conclave_config.with_sharemind_config(sharemind_conf)
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
conclave_config.input_path = os.path.join("/mnt/shared", data_root)
conclave_config.output_path = os.path.join("/mnt/shared", data_root)
suffix = "left" if pid == "1" else "right"
job_queue = generate_code(lambda: protocol_local(suffix, int(pid)),
conclave_config, ["sharemind"], ["python"],
apply_optimizations=False)
dispatch_jobs(job_queue, conclave_config)
def main():
pid = sys.argv[1]
data_root = sys.argv[2]
mpc_backend = sys.argv[3]
# define name for the workflow
workflow_name = "aspirin-large-join-" + pid + "-" + data_root
# configure conclave
conclave_config = CodeGenConfig(workflow_name, int(pid))
if mpc_backend == "sharemind":
sharemind_conf = SharemindCodeGenConfig("/mnt/shared",
use_docker=True,
use_hdfs=False)
conclave_config.with_sharemind_config(sharemind_conf)
elif mpc_backend == "obliv-c":
conclave_config.all_pids = [1, 2]
net_conf = [{
"host": "ca-spark-node-0",
"port": 8001
}, {
"host": "cb-spark-node-0",
"port": 8002
}]
net = NetworkConfig(net_conf, int(pid))
conclave_config.with_network_config(net)
oc_conf = OblivcConfig("/obliv-c/bin/oblivcc", "ca-spark-node-0:9000")
conclave_config.with_oc_config(oc_conf)
else:
raise Exception("Unknown MPC backend {}".format(mpc_backend))
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
conclave_config.input_path = os.path.join("/mnt/shared", data_root)
conclave_config.output_path = os.path.join("/mnt/shared", data_root)
job_queue = generate_code(protocol,
conclave_config, [mpc_backend], ["python"],
apply_optimizations=True)
dispatch_jobs(job_queue, conclave_config)
joined = cc.join(left, right, "joined", ["a"], ["c"])
cc.aggregate(joined, "expected", ["b"], "d", "sum", "total")
return {left, right}
if __name__ == "__main__":
pid = sys.argv[1]
# define name for the workflow
workflow_name = "simple-oblivious-test-" + pid
# configure conclave
conclave_config = CodeGenConfig(workflow_name, int(pid))
conclave_config.all_pids = [1]
sharemind_conf = SharemindCodeGenConfig("/mnt/shared",
use_docker=False,
use_hdfs=False)
conclave_config.with_sharemind_config(sharemind_conf)
current_dir = os.path.dirname(os.path.realpath(__file__))
# point conclave to the directory where the generated code should be stored/ read from
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
# point conclave to directory where data is to be read from...
conclave_config.input_path = os.path.join(current_dir, "data")
# and written to
conclave_config.output_path = os.path.join(current_dir, "data")
# define this party's unique ID (in this demo there is only one party)
job_queue = generate_code(protocol,
conclave_config, ["sharemind"], ["python"],
apply_optimizations=False)
dispatch_jobs(job_queue, conclave_config)
heart_patients = cc.cc_filter(aspirin,
"heart_patients",
diag_col_diags,
"==",
scalar=1)
cc.collect(cc.distinct_count(heart_patients, "actual", pid_col_meds), 1)
return {left_medication, left_diagnosis, right_medication, right_diagnosis}
if __name__ == "__main__":
pid = sys.argv[1]
# define name for the workflow
workflow_name = "real-aspirin-test-" + pid
# configure conclave
mpc_backend = sys.argv[2]
conclave_config = CodeGenConfig(workflow_name, int(pid)) \
.with_default_mpc_config(mpc_backend)
current_dir = os.path.dirname(os.path.realpath(__file__))
# point conclave to the directory where the generated code should be stored/ read from
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
# point conclave to directory where data is to be read from...
conclave_config.input_path = os.path.join(current_dir, "data")
# and written to
conclave_config.output_path = os.path.join(current_dir, "data")
job_queue = generate_code(lambda: protocol(conclave_config.all_pids),
conclave_config, [mpc_backend], ["python"],
apply_optimizations=True)
dispatch_jobs(job_queue, conclave_config)
def testHybridJoinWorkflow():
def protocol():
# define inputs
colsIn1 = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsIn1, set([1]))
proj1 = sal.project(in1, "proj1", ["a", "b"])
colsIn2 = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsIn2, set([2]))
proj2 = sal.project(in2, "proj2", ["c", "d"])
res = sal.join(proj1, proj2, "res", ["a"], ["c"])
# open result to party 1
sal.collect(res, 1)
# return roots of dag
return set([in1, in2])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "hybrid-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False,
use_docker=True)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.pid = pid
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/hybridjoin/" + size
codegen_config.output_path = "/mnt/shared/hybridjoin/" + size
jobqueue = generate_code(protocol, codegen_config, ["sharemind"],
["python"])
sharemind_config = {
"pid": pid,
"parties": {
1: {
"host": "localhost",
"port": 9001
},
2: {
"host": "localhost",
"port": 9002
},
3: {
"host": "localhost",
"port": 9003
}
}
}
sm_peer = setup_peer(sharemind_config)
conclave.dispatch.dispatch_all(None, sm_peer, jobqueue)
defCol("column_a", "INTEGER", [2]),
defCol("column_b", "INTEGER", [2])
]
right = cc.create("right", input_columns_right, {2})
rel = cc.concat([left, right], "rel")
filtered = cc.cc_filter(rel, "filtered", "column_b", "==", scalar=1)
in_order = cc.sort_by(filtered, "in_order", "column_a")
actual = cc.distinct_count(in_order, "actual", "column_a")
cc.collect(actual, 1)
return {left, right}
if __name__ == "__main__":
pid = sys.argv[1]
# define name for the workflow
workflow_name = "real-distinct-count-test-" + pid
# configure conclave
mpc_backend = sys.argv[2]
conclave_config = CodeGenConfig(workflow_name, int(pid)) \
.with_default_mpc_config(mpc_backend)
current_dir = os.path.dirname(os.path.realpath(__file__))
# point conclave to the directory where the generated code should be stored/ read from
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
# point conclave to directory where data is to be read from...
conclave_config.input_path = os.path.join(current_dir, "data")
# and written to
conclave_config.output_path = os.path.join(current_dir, "data")
# define this party's unique ID (in this demo there is only one party)
job_queue = generate_code(protocol, conclave_config, [mpc_backend], ["python"], apply_optimizations=True)
dispatch_jobs(job_queue, conclave_config)
